20:27:37.556 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
20:27:37.556 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taska94UBC/test-tmp
20:27:37.557 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/asan/bin
20:27:37.557 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taska94UBC/java/build/dist-test/../../../build/asan/bin/kudu, test, mini_cluster, --serialization=pb]
20:27:37.559 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-taska94UBC/test-tmp/mini-kudu-cluster1402564250707774898"
extra_master_flags: "--txn_manager_enabled"
extra_master_flags: "--raft_heartbeat_interval_ms=100"
extra_tserver_flags: "--txn_keepalive_interval_ms=1000"
extra_tserver_flags: "--txn_staleness_tracker_interval_ms=250"
extra_tserver_flags: "--enable_txn_system_client_init=true"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
20:27:37.894 [DEBUG - main] (MiniKuduCluster.java:195) Response:
20:27:37.894 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
20:27:37.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:37.897192 3256 mini_oidc.cc:150] Starting JWKS server
20:27:37.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:37.899641 3256 webserver.cc:492] Webserver started at http://127.0.0.1:33531/ using document root <none> and password file <none>
20:27:37.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:37.899878 3256 mini_oidc.cc:186] Starting OIDC Discovery server
20:27:37.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:37.900395 3256 webserver.cc:492] Webserver started at http://127.0.0.1:45773/ using document root <none> and password file <none>
20:27:37.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-05-01T20:27:37Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
20:27:37.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-05-01T20:27:37Z Disabled control of system clock
20:27:37.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:37.929637 3256 external_mini_cluster.cc:1366] Running /tmp/dist-test-taska94UBC/build/asan/bin/kudu
20:27:37.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taska94UBC/build/asan/bin/kudu
20:27:37.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taska94UBC/test-tmp/mini-kudu-cluster1402564250707774898/master-0/wal
20:27:37.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taska94UBC/test-tmp/mini-kudu-cluster1402564250707774898/master-0/data
20:27:37.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:27:37.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:27:37.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:27:37.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:27:37.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:27:37.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:27:37.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taska94UBC/test-tmp/mini-kudu-cluster1402564250707774898/master-0/logs
20:27:37.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taska94UBC/test-tmp/mini-kudu-cluster1402564250707774898/master-0/data/info.pb
20:27:37.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:27:37.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:27:37.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:27:37.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:27:37.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:27:37.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:27:37.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:27:37.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:27:37.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:27:37.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:27:37.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:27:37.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:27:37.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.46.62:46071
20:27:37.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.46.62
20:27:37.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:27:37.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.46.20:45141
20:27:37.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:27:37.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:27:37.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:27:37.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:27:37.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127
20:27:37.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:27:37.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33531/jwks.json
20:27:37.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled
20:27:37.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100 with env {}
20:27:38.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:27:38.260596 3267 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
20:27:38.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:27:38.260982 3267 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:27:38.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:27:38.261098 3267 flags.cc:432] Enabled unsafe flag: --never_fsync=true
20:27:38.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:27:38.270030 3267 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
20:27:38.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:27:38.270149 3267 flags.cc:432] Enabled experimental flag: --txn_manager_enabled=true
20:27:38.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:27:38.270220 3267 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
20:27:38.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:27:38.270277 3267 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
20:27:38.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:27:38.270354 3267 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
20:27:38.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:27:38.270413 3267 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
20:27:38.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:27:38.270483 3267 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:33531/jwks.json
20:27:38.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:38.281549 3267 master_runner.cc:386] Master server non-default flags:
20:27:38.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:27:38.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.46.20:45141
20:27:38.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:27:38.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:27:38.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100
20:27:38.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taska94UBC/test-tmp/mini-kudu-cluster1402564250707774898/master-0/data
20:27:38.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taska94UBC/test-tmp/mini-kudu-cluster1402564250707774898/master-0/wal
20:27:38.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:27:38.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127
20:27:38.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled=true
20:27:38.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:27:38.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:27:38.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:27:38.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.46.62:46071
20:27:38.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:27:38.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:27:38.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:27:38.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33531/jwks.json
20:27:38.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:27:38.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:27:38.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taska94UBC/test-tmp/mini-kudu-cluster1402564250707774898/master-0/data/info.pb
20:27:38.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.46.62
20:27:38.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:27:38.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:27:38.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:27:38.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:27:38.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:27:38.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:27:38.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taska94UBC/test-tmp/mini-kudu-cluster1402564250707774898/master-0/logs
20:27:38.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:27:38.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:27:38.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:27:38.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:27:38.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
20:27:38.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0ba7f54e15c9f673bf3e350f30b8d5f1a4037b79
20:27:38.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:27:38.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 01 May 2026 19:49:29 UTC on bdcb31816ec0
20:27:38.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11698
20:27:38.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
20:27:38.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:38.283476 3267 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:27:38.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:38.285346 3267 file_cache.cc:492] Constructed file cache file cache with capacity 419430
20:27:38.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:27:38.294370 3272 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:27:38.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:27:38.294673 3275 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:27:38.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:27:38.294688 3273 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:27:38.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:38.295914 3267 server_base.cc:1061] running on GCE node
20:27:38.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:38.296746 3267 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:27:38.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:38.298316 3267 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:27:38.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:38.299588 3267 hybrid_clock.cc:648] HybridClock initialized: now 1777667258299521 us; error 49 us; skew 500 ppm
20:27:38.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:38.299929 3267 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:27:38.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:38.302462 3267 webserver.cc:492] Webserver started at http://127.3.46.62:33105/ using document root <none> and password file <none>
20:27:38.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:38.303180 3267 fs_manager.cc:362] Metadata directory not provided
20:27:38.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:38.303311 3267 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:27:38.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:38.303608 3267 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
20:27:38.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:38.306203 3267 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taska94UBC/test-tmp/mini-kudu-cluster1402564250707774898/master-0/data/instance:
20:27:38.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0db1d539946942f794b0c4ec81f8f4b4"
20:27:38.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-01 20:27:38 on dist-test-slave-f471"
20:27:38.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:38.306981 3267 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taska94UBC/test-tmp/mini-kudu-cluster1402564250707774898/master-0/wal/instance:
20:27:38.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0db1d539946942f794b0c4ec81f8f4b4"
20:27:38.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-01 20:27:38 on dist-test-slave-f471"
20:27:38.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:38.312561 3267 fs_manager.cc:696] Time spent creating directory manager: real 0.005s user 0.002s sys 0.003s
20:27:38.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:38.316238 3281 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:27:38.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:38.317813 3267 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.003s sys 0.000s
20:27:38.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:38.317972 3267 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taska94UBC/test-tmp/mini-kudu-cluster1402564250707774898/master-0/data,/tmp/dist-test-taska94UBC/test-tmp/mini-kudu-cluster1402564250707774898/master-0/wal
20:27:38.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0db1d539946942f794b0c4ec81f8f4b4"
20:27:38.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-01 20:27:38 on dist-test-slave-f471"
20:27:38.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:38.318147 3267 fs_report.cc:389] FS layout report
20:27:38.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:27:38.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taska94UBC/test-tmp/mini-kudu-cluster1402564250707774898/master-0/wal
20:27:38.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taska94UBC/test-tmp/mini-kudu-cluster1402564250707774898/master-0/wal
20:27:38.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taska94UBC/test-tmp/mini-kudu-cluster1402564250707774898/master-0/data/data
20:27:38.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:27:38.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:27:38.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:27:38.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:27:38.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:27:38.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:27:38.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:27:38.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:27:38.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:27:38.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:27:38.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:27:38.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:38.333550 3267 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
20:27:38.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:38.334446 3267 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:27:38.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:38.334694 3267 kserver.cc:163] Server-wide thread pool size limit: 3276
20:27:38.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:38.357039 3267 rpc_server.cc:307] RPC server started. Bound to: 127.3.46.62:46071
20:27:38.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:38.357072 3342 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.3.46.62:46071 every 8 connection(s)
20:27:38.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:38.358603 3267 server_base.cc:1193] Dumped server information to /tmp/dist-test-taska94UBC/test-tmp/mini-kudu-cluster1402564250707774898/master-0/data/info.pb
20:27:38.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:38.362994 3343 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
20:27:38.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:38.367841 3256 external_mini_cluster.cc:1428] Started /tmp/dist-test-taska94UBC/build/asan/bin/kudu as pid 3267
20:27:38.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:38.368162 3256 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taska94UBC/test-tmp/mini-kudu-cluster1402564250707774898/master-0/wal/instance
20:27:38.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:38.368840 3343 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.3.46.62" port: 46071 } has no permanent_uuid. Determining permanent_uuid...
20:27:38.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:38.370543 3256 external_mini_cluster.cc:1366] Running /tmp/dist-test-taska94UBC/build/asan/bin/kudu
20:27:38.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taska94UBC/build/asan/bin/kudu
20:27:38.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taska94UBC/test-tmp/mini-kudu-cluster1402564250707774898/master-1/wal
20:27:38.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taska94UBC/test-tmp/mini-kudu-cluster1402564250707774898/master-1/data
20:27:38.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:27:38.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:27:38.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:27:38.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:27:38.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:27:38.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:27:38.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taska94UBC/test-tmp/mini-kudu-cluster1402564250707774898/master-1/logs
20:27:38.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taska94UBC/test-tmp/mini-kudu-cluster1402564250707774898/master-1/data/info.pb
20:27:38.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:27:38.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:27:38.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:27:38.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:27:38.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:27:38.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:27:38.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:27:38.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:27:38.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:27:38.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:27:38.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:27:38.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:27:38.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.46.61:39407
20:27:38.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.46.61
20:27:38.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:27:38.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.46.20:45141
20:27:38.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:27:38.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:27:38.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:27:38.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:27:38.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127
20:27:38.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:27:38.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33531/jwks.json
20:27:38.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled
20:27:38.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100 with env {}
20:27:38.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:38.385515 3343 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.3.46.61" port: 39407 } has no permanent_uuid. Determining permanent_uuid...
20:27:38.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:27:38.386729 3283 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.3.46.61:39407: connect: Connection refused (error 111)
20:27:38.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:27:38.388378 3343 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.3.46.61:39407: Network error: Client connection negotiation failed: client connection to 127.3.46.61:39407: connect: Connection refused (error 111)
20:27:38.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:38.437829 3343 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.46.61" port: 39407 } attempt: 1
20:27:38.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:27:38.439917 3343 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.3.46.61:39407: Network error: Client connection negotiation failed: client connection to 127.3.46.61:39407: connect: Connection refused (error 111)
20:27:38.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:38.508251 3343 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.46.61" port: 39407 } attempt: 2
20:27:38.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:27:38.510205 3343 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.3.46.61:39407: Network error: Client connection negotiation failed: client connection to 127.3.46.61:39407: connect: Connection refused (error 111)
20:27:38.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:38.601536 3343 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.46.61" port: 39407 } attempt: 3
20:27:38.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:27:38.603646 3343 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.3.46.61:39407: Network error: Client connection negotiation failed: client connection to 127.3.46.61:39407: connect: Connection refused (error 111)
20:27:38.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:27:38.707747 3347 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
20:27:38.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:27:38.708076 3347 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:27:38.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:27:38.708151 3347 flags.cc:432] Enabled unsafe flag: --never_fsync=true
20:27:38.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:27:38.717074 3347 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
20:27:38.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:27:38.717206 3347 flags.cc:432] Enabled experimental flag: --txn_manager_enabled=true
20:27:38.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:27:38.717274 3347 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
20:27:38.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:27:38.717315 3347 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
20:27:38.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:27:38.717370 3347 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
20:27:38.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:27:38.717408 3347 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
20:27:38.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:27:38.717450 3347 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:33531/jwks.json
20:27:38.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:38.728410 3347 master_runner.cc:386] Master server non-default flags:
20:27:38.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:27:38.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.46.20:45141
20:27:38.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:27:38.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:27:38.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100
20:27:38.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taska94UBC/test-tmp/mini-kudu-cluster1402564250707774898/master-1/data
20:27:38.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taska94UBC/test-tmp/mini-kudu-cluster1402564250707774898/master-1/wal
20:27:38.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:27:38.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127
20:27:38.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled=true
20:27:38.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:27:38.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:27:38.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:27:38.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.46.61:39407
20:27:38.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:27:38.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:27:38.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:27:38.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33531/jwks.json
20:27:38.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:27:38.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:27:38.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taska94UBC/test-tmp/mini-kudu-cluster1402564250707774898/master-1/data/info.pb
20:27:38.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.46.61
20:27:38.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:27:38.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:27:38.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:27:38.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:27:38.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:27:38.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:27:38.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taska94UBC/test-tmp/mini-kudu-cluster1402564250707774898/master-1/logs
20:27:38.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:27:38.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:27:38.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:27:38.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:27:38.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
20:27:38.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0ba7f54e15c9f673bf3e350f30b8d5f1a4037b79
20:27:38.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:27:38.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 01 May 2026 19:49:29 UTC on bdcb31816ec0
20:27:38.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11698
20:27:38.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
20:27:38.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:38.730141 3347 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:27:38.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:38.731900 3347 file_cache.cc:492] Constructed file cache file cache with capacity 419430
20:27:38.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:27:38.740036 3353 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:27:38.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:27:38.740074 3354 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:27:38.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:27:38.740724 3356 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:27:38.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:38.741097 3347 server_base.cc:1061] running on GCE node
20:27:38.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:38.741811 3347 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:27:38.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:38.743193 3347 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:27:38.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:38.744411 3347 hybrid_clock.cc:648] HybridClock initialized: now 1777667258744353 us; error 32 us; skew 500 ppm
20:27:38.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:38.744742 3347 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:27:38.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:38.747027 3347 webserver.cc:492] Webserver started at http://127.3.46.61:45983/ using document root <none> and password file <none>
20:27:38.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:38.746943 3343 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.46.61" port: 39407 } attempt: 4
20:27:38.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:38.747747 3347 fs_manager.cc:362] Metadata directory not provided
20:27:38.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:38.747852 3347 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:27:38.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:38.748215 3347 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
20:27:38.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:27:38.748924 3343 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.3.46.61:39407: Network error: Client connection negotiation failed: client connection to 127.3.46.61:39407: connect: Connection refused (error 111)
20:27:38.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:38.750804 3347 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taska94UBC/test-tmp/mini-kudu-cluster1402564250707774898/master-1/data/instance:
20:27:38.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d6c270b387b04f17b2a58bfcb0eb4f82"
20:27:38.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-01 20:27:38 on dist-test-slave-f471"
20:27:38.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:38.751564 3347 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taska94UBC/test-tmp/mini-kudu-cluster1402564250707774898/master-1/wal/instance:
20:27:38.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d6c270b387b04f17b2a58bfcb0eb4f82"
20:27:38.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-01 20:27:38 on dist-test-slave-f471"
20:27:38.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:38.756939 3347 fs_manager.cc:696] Time spent creating directory manager: real 0.005s user 0.006s sys 0.000s
20:27:38.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:38.760679 3362 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:27:38.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:38.762254 3347 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.003s sys 0.000s
20:27:38.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:38.762456 3347 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taska94UBC/test-tmp/mini-kudu-cluster1402564250707774898/master-1/data,/tmp/dist-test-taska94UBC/test-tmp/mini-kudu-cluster1402564250707774898/master-1/wal
20:27:38.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d6c270b387b04f17b2a58bfcb0eb4f82"
20:27:38.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-01 20:27:38 on dist-test-slave-f471"
20:27:38.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:38.762658 3347 fs_report.cc:389] FS layout report
20:27:38.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:27:38.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taska94UBC/test-tmp/mini-kudu-cluster1402564250707774898/master-1/wal
20:27:38.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taska94UBC/test-tmp/mini-kudu-cluster1402564250707774898/master-1/wal
20:27:38.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taska94UBC/test-tmp/mini-kudu-cluster1402564250707774898/master-1/data/data
20:27:38.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:27:38.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:27:38.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:27:38.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:27:38.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:27:38.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:27:38.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:27:38.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:27:38.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:27:38.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:27:38.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:27:38.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:38.777594 3347 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
20:27:38.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:38.778541 3347 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:27:38.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:38.778818 3347 kserver.cc:163] Server-wide thread pool size limit: 3276
20:27:38.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:38.802346 3347 rpc_server.cc:307] RPC server started. Bound to: 127.3.46.61:39407
20:27:38.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:38.802384 3423 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.3.46.61:39407 every 8 connection(s)
20:27:38.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:38.803875 3347 server_base.cc:1193] Dumped server information to /tmp/dist-test-taska94UBC/test-tmp/mini-kudu-cluster1402564250707774898/master-1/data/info.pb
20:27:38.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:38.808456 3424 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
20:27:38.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:38.810688 3256 external_mini_cluster.cc:1428] Started /tmp/dist-test-taska94UBC/build/asan/bin/kudu as pid 3347
20:27:38.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:38.810853 3256 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taska94UBC/test-tmp/mini-kudu-cluster1402564250707774898/master-1/wal/instance
20:27:38.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:38.812099 3256 external_mini_cluster.cc:1366] Running /tmp/dist-test-taska94UBC/build/asan/bin/kudu
20:27:38.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taska94UBC/build/asan/bin/kudu
20:27:38.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taska94UBC/test-tmp/mini-kudu-cluster1402564250707774898/master-2/wal
20:27:38.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taska94UBC/test-tmp/mini-kudu-cluster1402564250707774898/master-2/data
20:27:38.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:27:38.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:27:38.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:27:38.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:27:38.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:27:38.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:27:38.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taska94UBC/test-tmp/mini-kudu-cluster1402564250707774898/master-2/logs
20:27:38.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taska94UBC/test-tmp/mini-kudu-cluster1402564250707774898/master-2/data/info.pb
20:27:38.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:27:38.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:27:38.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:27:38.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:27:38.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:27:38.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:27:38.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:27:38.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:27:38.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:27:38.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:27:38.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:27:38.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:27:38.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.46.60:36127
20:27:38.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.46.60
20:27:38.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:27:38.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.46.20:45141
20:27:38.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:27:38.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:27:38.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:27:38.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:27:38.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127
20:27:38.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:27:38.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33531/jwks.json
20:27:38.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled
20:27:38.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100 with env {}
20:27:38.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:38.815852 3424 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.3.46.62" port: 46071 } has no permanent_uuid. Determining permanent_uuid...
20:27:38.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:38.829784 3424 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.3.46.61" port: 39407 } has no permanent_uuid. Determining permanent_uuid...
20:27:38.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:38.839128 3424 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.3.46.60" port: 36127 } has no permanent_uuid. Determining permanent_uuid...
20:27:38.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:27:38.840299 3363 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.3.46.60:36127: connect: Connection refused (error 111)
20:27:38.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:27:38.841934 3424 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.3.46.60:36127: Network error: Client connection negotiation failed: client connection to 127.3.46.60:36127: connect: Connection refused (error 111)
20:27:38.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:38.891239 3424 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.46.60" port: 36127 } attempt: 1
20:27:38.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:27:38.893612 3424 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.3.46.60:36127: Network error: Client connection negotiation failed: client connection to 127.3.46.60:36127: connect: Connection refused (error 111)
20:27:38.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:38.961803 3424 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.46.60" port: 36127 } attempt: 2
20:27:38.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:27:38.963855 3424 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.3.46.60:36127: Network error: Client connection negotiation failed: client connection to 127.3.46.60:36127: connect: Connection refused (error 111)
20:27:39.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:39.048208 3343 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.46.61" port: 39407 } attempt: 5
20:27:39.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:39.055099 3424 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.46.60" port: 36127 } attempt: 3
20:27:39.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:39.055464 3343 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.3.46.60" port: 36127 } has no permanent_uuid. Determining permanent_uuid...
20:27:39.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:27:39.057451 3424 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.3.46.60:36127: Network error: Client connection negotiation failed: client connection to 127.3.46.60:36127: connect: Connection refused (error 111)
20:27:39.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:27:39.057446 3343 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.3.46.60:36127: Network error: Client connection negotiation failed: client connection to 127.3.46.60:36127: connect: Connection refused (error 111)
20:27:39.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:39.108673 3343 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.46.60" port: 36127 } attempt: 1
20:27:39.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:27:39.110605 3343 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.3.46.60:36127: Network error: Client connection negotiation failed: client connection to 127.3.46.60:36127: connect: Connection refused (error 111)
20:27:39.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:27:39.163683 3426 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
20:27:39.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:27:39.164022 3426 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:27:39.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:27:39.164099 3426 flags.cc:432] Enabled unsafe flag: --never_fsync=true
20:27:39.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:27:39.173362 3426 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
20:27:39.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:27:39.173451 3426 flags.cc:432] Enabled experimental flag: --txn_manager_enabled=true
20:27:39.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:27:39.173497 3426 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
20:27:39.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:27:39.173537 3426 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
20:27:39.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:27:39.173590 3426 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
20:27:39.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:27:39.173627 3426 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
20:27:39.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:27:39.173671 3426 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:33531/jwks.json
20:27:39.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:39.178866 3343 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.46.60" port: 36127 } attempt: 2
20:27:39.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:27:39.180996 3343 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.3.46.60:36127: Network error: Client connection negotiation failed: client connection to 127.3.46.60:36127: connect: Connection refused (error 111)
20:27:39.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:39.185256 3426 master_runner.cc:386] Master server non-default flags:
20:27:39.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:27:39.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.46.20:45141
20:27:39.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:27:39.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:27:39.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100
20:27:39.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taska94UBC/test-tmp/mini-kudu-cluster1402564250707774898/master-2/data
20:27:39.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taska94UBC/test-tmp/mini-kudu-cluster1402564250707774898/master-2/wal
20:27:39.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:27:39.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127
20:27:39.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled=true
20:27:39.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:27:39.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:27:39.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:27:39.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.46.60:36127
20:27:39.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:27:39.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:27:39.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:27:39.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33531/jwks.json
20:27:39.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:27:39.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:27:39.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taska94UBC/test-tmp/mini-kudu-cluster1402564250707774898/master-2/data/info.pb
20:27:39.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.46.60
20:27:39.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:27:39.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:27:39.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:27:39.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:27:39.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:27:39.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:27:39.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taska94UBC/test-tmp/mini-kudu-cluster1402564250707774898/master-2/logs
20:27:39.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:27:39.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:27:39.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:27:39.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:27:39.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
20:27:39.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0ba7f54e15c9f673bf3e350f30b8d5f1a4037b79
20:27:39.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:27:39.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 01 May 2026 19:49:29 UTC on bdcb31816ec0
20:27:39.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11698
20:27:39.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
20:27:39.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:39.187286 3426 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:27:39.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:39.189174 3426 file_cache.cc:492] Constructed file cache file cache with capacity 419430
20:27:39.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:27:39.198213 3434 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:27:39.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:27:39.198329 3435 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:27:39.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:27:39.198525 3437 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:27:39.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:39.199657 3426 server_base.cc:1061] running on GCE node
20:27:39.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:39.200587 3426 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:27:39.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:39.200672 3424 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.46.60" port: 36127 } attempt: 4
20:27:39.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:39.202363 3426 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:27:39.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:27:39.202476 3424 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.3.46.60:36127: Network error: Client connection negotiation failed: client connection to 127.3.46.60:36127: connect: Connection refused (error 111)
20:27:39.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:39.203635 3426 hybrid_clock.cc:648] HybridClock initialized: now 1777667259203554 us; error 46 us; skew 500 ppm
20:27:39.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:39.203979 3426 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:27:39.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:39.207122 3426 webserver.cc:492] Webserver started at http://127.3.46.60:38161/ using document root <none> and password file <none>
20:27:39.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:39.207867 3426 fs_manager.cc:362] Metadata directory not provided
20:27:39.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:39.207988 3426 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:27:39.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:39.208339 3426 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
20:27:39.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:39.211037 3426 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taska94UBC/test-tmp/mini-kudu-cluster1402564250707774898/master-2/data/instance:
20:27:39.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b1731f8308c145f686f88efd6a54b851"
20:27:39.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-01 20:27:39 on dist-test-slave-f471"
20:27:39.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:39.211880 3426 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taska94UBC/test-tmp/mini-kudu-cluster1402564250707774898/master-2/wal/instance:
20:27:39.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b1731f8308c145f686f88efd6a54b851"
20:27:39.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-01 20:27:39 on dist-test-slave-f471"
20:27:39.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:39.217687 3426 fs_manager.cc:696] Time spent creating directory manager: real 0.005s user 0.002s sys 0.005s
20:27:39.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:39.222168 3443 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:27:39.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:39.223912 3426 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.004s sys 0.000s
20:27:39.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:39.224114 3426 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taska94UBC/test-tmp/mini-kudu-cluster1402564250707774898/master-2/data,/tmp/dist-test-taska94UBC/test-tmp/mini-kudu-cluster1402564250707774898/master-2/wal
20:27:39.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b1731f8308c145f686f88efd6a54b851"
20:27:39.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-01 20:27:39 on dist-test-slave-f471"
20:27:39.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:39.224310 3426 fs_report.cc:389] FS layout report
20:27:39.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:27:39.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taska94UBC/test-tmp/mini-kudu-cluster1402564250707774898/master-2/wal
20:27:39.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taska94UBC/test-tmp/mini-kudu-cluster1402564250707774898/master-2/wal
20:27:39.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taska94UBC/test-tmp/mini-kudu-cluster1402564250707774898/master-2/data/data
20:27:39.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:27:39.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:27:39.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:27:39.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:27:39.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:27:39.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:27:39.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:27:39.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:27:39.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:27:39.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:27:39.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:27:39.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:39.245817 3426 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
20:27:39.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:39.246707 3426 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:27:39.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:39.246973 3426 kserver.cc:163] Server-wide thread pool size limit: 3276
20:27:39.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:39.272383 3426 rpc_server.cc:307] RPC server started. Bound to: 127.3.46.60:36127
20:27:39.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:39.272418 3504 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.3.46.60:36127 every 8 connection(s)
20:27:39.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:39.273955 3426 server_base.cc:1193] Dumped server information to /tmp/dist-test-taska94UBC/test-tmp/mini-kudu-cluster1402564250707774898/master-2/data/info.pb
20:27:39.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:39.278070 3505 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
20:27:39.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:39.282269 3256 external_mini_cluster.cc:1428] Started /tmp/dist-test-taska94UBC/build/asan/bin/kudu as pid 3426
20:27:39.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:39.282438 3256 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taska94UBC/test-tmp/mini-kudu-cluster1402564250707774898/master-2/wal/instance
20:27:39.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:39.284305 3505 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.3.46.62" port: 46071 } has no permanent_uuid. Determining permanent_uuid...
20:27:39.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:39.287207 3343 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.46.60" port: 36127 } attempt: 3
20:27:39.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:39.302513 3505 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.3.46.61" port: 39407 } has no permanent_uuid. Determining permanent_uuid...
20:27:39.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:39.307307 3343 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 0db1d539946942f794b0c4ec81f8f4b4: Bootstrap starting.
20:27:39.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:39.311584 3343 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 0db1d539946942f794b0c4ec81f8f4b4: Neither blocks nor log segments found. Creating new log.
20:27:39.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:39.312012 3505 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.3.46.60" port: 36127 } has no permanent_uuid. Determining permanent_uuid...
20:27:39.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:39.313021 3343 log.cc:826] T 00000000000000000000000000000000 P 0db1d539946942f794b0c4ec81f8f4b4: Log is configured to *not* fsync() on all Append() calls
20:27:39.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:39.319988 3343 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 0db1d539946942f794b0c4ec81f8f4b4: No bootstrap required, opened a new log
20:27:39.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:39.322615 3505 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P b1731f8308c145f686f88efd6a54b851: Bootstrap starting.
20:27:39.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:39.325956 3343 raft_consensus.cc:359] T 00000000000000000000000000000000 P 0db1d539946942f794b0c4ec81f8f4b4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0db1d539946942f794b0c4ec81f8f4b4" member_type: VOTER last_known_addr { host: "127.3.46.62" port: 46071 } } peers { permanent_uuid: "d6c270b387b04f17b2a58bfcb0eb4f82" member_type: VOTER last_known_addr { host: "127.3.46.61" port: 39407 } } peers { permanent_uuid: "b1731f8308c145f686f88efd6a54b851" member_type: VOTER last_known_addr { host: "127.3.46.60" port: 36127 } }
20:27:39.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:39.326267 3343 raft_consensus.cc:385] T 00000000000000000000000000000000 P 0db1d539946942f794b0c4ec81f8f4b4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
20:27:39.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:39.326397 3505 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P b1731f8308c145f686f88efd6a54b851: Neither blocks nor log segments found. Creating new log.
20:27:39.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:39.326383 3343 raft_consensus.cc:740] T 00000000000000000000000000000000 P 0db1d539946942f794b0c4ec81f8f4b4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 0db1d539946942f794b0c4ec81f8f4b4, State: Initialized, Role: FOLLOWER
20:27:39.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:39.327014 3343 consensus_queue.cc:260] T 00000000000000000000000000000000 P 0db1d539946942f794b0c4ec81f8f4b4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0db1d539946942f794b0c4ec81f8f4b4" member_type: VOTER last_known_addr { host: "127.3.46.62" port: 46071 } } peers { permanent_uuid: "d6c270b387b04f17b2a58bfcb0eb4f82" member_type: VOTER last_known_addr { host: "127.3.46.61" port: 39407 } } peers { permanent_uuid: "b1731f8308c145f686f88efd6a54b851" member_type: VOTER last_known_addr { host: "127.3.46.60" port: 36127 } }
20:27:39.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:39.327410 3505 log.cc:826] T 00000000000000000000000000000000 P b1731f8308c145f686f88efd6a54b851: Log is configured to *not* fsync() on all Append() calls
20:27:39.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:39.328001 3514 sys_catalog.cc:455] T 00000000000000000000000000000000 P 0db1d539946942f794b0c4ec81f8f4b4 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0db1d539946942f794b0c4ec81f8f4b4" member_type: VOTER last_known_addr { host: "127.3.46.62" port: 46071 } } peers { permanent_uuid: "d6c270b387b04f17b2a58bfcb0eb4f82" member_type: VOTER last_known_addr { host: "127.3.46.61" port: 39407 } } peers { permanent_uuid: "b1731f8308c145f686f88efd6a54b851" member_type: VOTER last_known_addr { host: "127.3.46.60" port: 36127 } } }
20:27:39.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:39.328455 3514 sys_catalog.cc:458] T 00000000000000000000000000000000 P 0db1d539946942f794b0c4ec81f8f4b4 [sys.catalog]: This master's current role is: FOLLOWER
20:27:39.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:39.330273 3343 sys_catalog.cc:565] T 00000000000000000000000000000000 P 0db1d539946942f794b0c4ec81f8f4b4 [sys.catalog]: configured and running, proceeding with master startup.
20:27:39.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:39.331077 3505 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P b1731f8308c145f686f88efd6a54b851: No bootstrap required, opened a new log
20:27:39.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:39.337638 3505 raft_consensus.cc:359] T 00000000000000000000000000000000 P b1731f8308c145f686f88efd6a54b851 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0db1d539946942f794b0c4ec81f8f4b4" member_type: VOTER last_known_addr { host: "127.3.46.62" port: 46071 } } peers { permanent_uuid: "d6c270b387b04f17b2a58bfcb0eb4f82" member_type: VOTER last_known_addr { host: "127.3.46.61" port: 39407 } } peers { permanent_uuid: "b1731f8308c145f686f88efd6a54b851" member_type: VOTER last_known_addr { host: "127.3.46.60" port: 36127 } }
20:27:39.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:39.338191 3505 raft_consensus.cc:385] T 00000000000000000000000000000000 P b1731f8308c145f686f88efd6a54b851 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
20:27:39.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:39.338327 3505 raft_consensus.cc:740] T 00000000000000000000000000000000 P b1731f8308c145f686f88efd6a54b851 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: b1731f8308c145f686f88efd6a54b851, State: Initialized, Role: FOLLOWER
20:27:39.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:27:39.337606 3287 tablet.cc:2404] T 00000000000000000000000000000000 P 0db1d539946942f794b0c4ec81f8f4b4: Can't schedule compaction. Clean time has not been advanced past its initial value.
20:27:39.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:39.339376 3505 consensus_queue.cc:260] T 00000000000000000000000000000000 P b1731f8308c145f686f88efd6a54b851 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0db1d539946942f794b0c4ec81f8f4b4" member_type: VOTER last_known_addr { host: "127.3.46.62" port: 46071 } } peers { permanent_uuid: "d6c270b387b04f17b2a58bfcb0eb4f82" member_type: VOTER last_known_addr { host: "127.3.46.61" port: 39407 } } peers { permanent_uuid: "b1731f8308c145f686f88efd6a54b851" member_type: VOTER last_known_addr { host: "127.3.46.60" port: 36127 } }
20:27:39.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:39.340847 3527 sys_catalog.cc:455] T 00000000000000000000000000000000 P b1731f8308c145f686f88efd6a54b851 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0db1d539946942f794b0c4ec81f8f4b4" member_type: VOTER last_known_addr { host: "127.3.46.62" port: 46071 } } peers { permanent_uuid: "d6c270b387b04f17b2a58bfcb0eb4f82" member_type: VOTER last_known_addr { host: "127.3.46.61" port: 39407 } } peers { permanent_uuid: "b1731f8308c145f686f88efd6a54b851" member_type: VOTER last_known_addr { host: "127.3.46.60" port: 36127 } } }
20:27:39.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:39.341308 3527 sys_catalog.cc:458] T 00000000000000000000000000000000 P b1731f8308c145f686f88efd6a54b851 [sys.catalog]: This master's current role is: FOLLOWER
20:27:39.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:39.342233 3505 sys_catalog.cc:565] T 00000000000000000000000000000000 P b1731f8308c145f686f88efd6a54b851 [sys.catalog]: configured and running, proceeding with master startup.
20:27:39.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:27:39.344393 3526 catalog_manager.cc:1568] T 00000000000000000000000000000000 P 0db1d539946942f794b0c4ec81f8f4b4: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:27:39.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:27:39.344534 3526 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:27:39.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:39.343088 3267 master_runner.cc:186] Error getting master registration for 127.3.46.61:39407: OK, instance_id { permanent_uuid: "d6c270b387b04f17b2a58bfcb0eb4f82" instance_seqno: 1777667258796877 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
20:27:39.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:39.351483 3267 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
20:27:39.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:27:39.357172 3539 catalog_manager.cc:1568] T 00000000000000000000000000000000 P b1731f8308c145f686f88efd6a54b851: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:27:39.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:27:39.357550 3539 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:27:39.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:39.366084 3514 raft_consensus.cc:493] T 00000000000000000000000000000000 P 0db1d539946942f794b0c4ec81f8f4b4 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
20:27:39.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:39.366382 3514 raft_consensus.cc:515] T 00000000000000000000000000000000 P 0db1d539946942f794b0c4ec81f8f4b4 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0db1d539946942f794b0c4ec81f8f4b4" member_type: VOTER last_known_addr { host: "127.3.46.62" port: 46071 } } peers { permanent_uuid: "d6c270b387b04f17b2a58bfcb0eb4f82" member_type: VOTER last_known_addr { host: "127.3.46.61" port: 39407 } } peers { permanent_uuid: "b1731f8308c145f686f88efd6a54b851" member_type: VOTER last_known_addr { host: "127.3.46.60" port: 36127 } }
20:27:39.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:39.361034 3426 master_runner.cc:186] Error getting master registration for 127.3.46.61:39407: OK, instance_id { permanent_uuid: "d6c270b387b04f17b2a58bfcb0eb4f82" instance_seqno: 1777667258796877 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
20:27:39.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:39.367425 3426 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
20:27:39.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:39.368116 3514 leader_election.cc:290] T 00000000000000000000000000000000 P 0db1d539946942f794b0c4ec81f8f4b4 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers d6c270b387b04f17b2a58bfcb0eb4f82 (127.3.46.61:39407), b1731f8308c145f686f88efd6a54b851 (127.3.46.60:36127)
20:27:39.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:39.369442 3470 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "0db1d539946942f794b0c4ec81f8f4b4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "b1731f8308c145f686f88efd6a54b851" is_pre_election: true
20:27:39.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:39.370021 3470 raft_consensus.cc:2468] T 00000000000000000000000000000000 P b1731f8308c145f686f88efd6a54b851 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 0db1d539946942f794b0c4ec81f8f4b4 in term 0.
20:27:39.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:39.370709 3282 leader_election.cc:304] T 00000000000000000000000000000000 P 0db1d539946942f794b0c4ec81f8f4b4 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 0db1d539946942f794b0c4ec81f8f4b4, b1731f8308c145f686f88efd6a54b851; no voters:
20:27:39.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:39.371169 3514 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 0db1d539946942f794b0c4ec81f8f4b4 [term 0 FOLLOWER]: Leader pre-election won for term 1
20:27:39.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:39.371325 3514 raft_consensus.cc:493] T 00000000000000000000000000000000 P 0db1d539946942f794b0c4ec81f8f4b4 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
20:27:39.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:39.371454 3514 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 0db1d539946942f794b0c4ec81f8f4b4 [term 0 FOLLOWER]: Advancing to term 1
20:27:39.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:39.374752 3514 raft_consensus.cc:515] T 00000000000000000000000000000000 P 0db1d539946942f794b0c4ec81f8f4b4 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0db1d539946942f794b0c4ec81f8f4b4" member_type: VOTER last_known_addr { host: "127.3.46.62" port: 46071 } } peers { permanent_uuid: "d6c270b387b04f17b2a58bfcb0eb4f82" member_type: VOTER last_known_addr { host: "127.3.46.61" port: 39407 } } peers { permanent_uuid: "b1731f8308c145f686f88efd6a54b851" member_type: VOTER last_known_addr { host: "127.3.46.60" port: 36127 } }
20:27:39.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:39.377583 3514 leader_election.cc:290] T 00000000000000000000000000000000 P 0db1d539946942f794b0c4ec81f8f4b4 [CANDIDATE]: Term 1 election: Requested vote from peers d6c270b387b04f17b2a58bfcb0eb4f82 (127.3.46.61:39407), b1731f8308c145f686f88efd6a54b851 (127.3.46.60:36127)
20:27:39.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:39.377544 3470 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "0db1d539946942f794b0c4ec81f8f4b4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "b1731f8308c145f686f88efd6a54b851"
20:27:39.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:39.377790 3470 raft_consensus.cc:3060] T 00000000000000000000000000000000 P b1731f8308c145f686f88efd6a54b851 [term 0 FOLLOWER]: Advancing to term 1
20:27:39.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:39.381240 3470 raft_consensus.cc:2468] T 00000000000000000000000000000000 P b1731f8308c145f686f88efd6a54b851 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 0db1d539946942f794b0c4ec81f8f4b4 in term 1.
20:27:39.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:39.381774 3282 leader_election.cc:304] T 00000000000000000000000000000000 P 0db1d539946942f794b0c4ec81f8f4b4 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 0db1d539946942f794b0c4ec81f8f4b4, b1731f8308c145f686f88efd6a54b851; no voters:
20:27:39.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:39.382215 3514 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 0db1d539946942f794b0c4ec81f8f4b4 [term 1 FOLLOWER]: Leader election won for term 1
20:27:39.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:39.382858 3514 raft_consensus.cc:697] T 00000000000000000000000000000000 P 0db1d539946942f794b0c4ec81f8f4b4 [term 1 LEADER]: Becoming Leader. State: Replica: 0db1d539946942f794b0c4ec81f8f4b4, State: Running, Role: LEADER
20:27:39.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:39.383461 3514 consensus_queue.cc:237] T 00000000000000000000000000000000 P 0db1d539946942f794b0c4ec81f8f4b4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0db1d539946942f794b0c4ec81f8f4b4" member_type: VOTER last_known_addr { host: "127.3.46.62" port: 46071 } } peers { permanent_uuid: "d6c270b387b04f17b2a58bfcb0eb4f82" member_type: VOTER last_known_addr { host: "127.3.46.61" port: 39407 } } peers { permanent_uuid: "b1731f8308c145f686f88efd6a54b851" member_type: VOTER last_known_addr { host: "127.3.46.60" port: 36127 } }
20:27:39.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:39.386986 3541 sys_catalog.cc:455] T 00000000000000000000000000000000 P 0db1d539946942f794b0c4ec81f8f4b4 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 0db1d539946942f794b0c4ec81f8f4b4. Latest consensus state: current_term: 1 leader_uuid: "0db1d539946942f794b0c4ec81f8f4b4" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0db1d539946942f794b0c4ec81f8f4b4" member_type: VOTER last_known_addr { host: "127.3.46.62" port: 46071 } } peers { permanent_uuid: "d6c270b387b04f17b2a58bfcb0eb4f82" member_type: VOTER last_known_addr { host: "127.3.46.61" port: 39407 } } peers { permanent_uuid: "b1731f8308c145f686f88efd6a54b851" member_type: VOTER last_known_addr { host: "127.3.46.60" port: 36127 } } }
20:27:39.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:39.387382 3541 sys_catalog.cc:458] T 00000000000000000000000000000000 P 0db1d539946942f794b0c4ec81f8f4b4 [sys.catalog]: This master's current role is: LEADER
20:27:39.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:39.397398 3543 catalog_manager.cc:1485] Loading table and tablet metadata into memory...
20:27:39.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:39.399811 3543 catalog_manager.cc:1494] Initializing Kudu cluster ID...
20:27:39.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:39.416145 3470 raft_consensus.cc:1275] T 00000000000000000000000000000000 P b1731f8308c145f686f88efd6a54b851 [term 1 FOLLOWER]: Refusing update from remote peer 0db1d539946942f794b0c4ec81f8f4b4: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
20:27:39.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:39.417479 3541 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 0db1d539946942f794b0c4ec81f8f4b4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "b1731f8308c145f686f88efd6a54b851" member_type: VOTER last_known_addr { host: "127.3.46.60" port: 36127 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
20:27:39.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:39.437866 3527 sys_catalog.cc:455] T 00000000000000000000000000000000 P b1731f8308c145f686f88efd6a54b851 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 0db1d539946942f794b0c4ec81f8f4b4. Latest consensus state: current_term: 1 leader_uuid: "0db1d539946942f794b0c4ec81f8f4b4" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0db1d539946942f794b0c4ec81f8f4b4" member_type: VOTER last_known_addr { host: "127.3.46.62" port: 46071 } } peers { permanent_uuid: "d6c270b387b04f17b2a58bfcb0eb4f82" member_type: VOTER last_known_addr { host: "127.3.46.61" port: 39407 } } peers { permanent_uuid: "b1731f8308c145f686f88efd6a54b851" member_type: VOTER last_known_addr { host: "127.3.46.60" port: 36127 } } }
20:27:39.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:39.438182 3527 sys_catalog.cc:458] T 00000000000000000000000000000000 P b1731f8308c145f686f88efd6a54b851 [sys.catalog]: This master's current role is: FOLLOWER
20:27:39.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:39.445529 3544 mvcc.cc:204] Tried to move back new op lower bound from 7281325094550441984 to 7281325094439702528. Current Snapshot: MvccSnapshot[applied={T|T < 7281325094550441984}]
20:27:39.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:39.448841 3545 mvcc.cc:204] Tried to move back new op lower bound from 7281325094550441984 to 7281325094439702528. Current Snapshot: MvccSnapshot[applied={T|T < 7281325094550441984}]
20:27:39.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:39.449373 3514 sys_catalog.cc:455] T 00000000000000000000000000000000 P 0db1d539946942f794b0c4ec81f8f4b4 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "0db1d539946942f794b0c4ec81f8f4b4" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0db1d539946942f794b0c4ec81f8f4b4" member_type: VOTER last_known_addr { host: "127.3.46.62" port: 46071 } } peers { permanent_uuid: "d6c270b387b04f17b2a58bfcb0eb4f82" member_type: VOTER last_known_addr { host: "127.3.46.61" port: 39407 } } peers { permanent_uuid: "b1731f8308c145f686f88efd6a54b851" member_type: VOTER last_known_addr { host: "127.3.46.60" port: 36127 } } }
20:27:39.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:39.450156 3514 sys_catalog.cc:458] T 00000000000000000000000000000000 P 0db1d539946942f794b0c4ec81f8f4b4 [sys.catalog]: This master's current role is: LEADER
20:27:39.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:39.458659 3527 sys_catalog.cc:455] T 00000000000000000000000000000000 P b1731f8308c145f686f88efd6a54b851 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "0db1d539946942f794b0c4ec81f8f4b4" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0db1d539946942f794b0c4ec81f8f4b4" member_type: VOTER last_known_addr { host: "127.3.46.62" port: 46071 } } peers { permanent_uuid: "d6c270b387b04f17b2a58bfcb0eb4f82" member_type: VOTER last_known_addr { host: "127.3.46.61" port: 39407 } } peers { permanent_uuid: "b1731f8308c145f686f88efd6a54b851" member_type: VOTER last_known_addr { host: "127.3.46.60" port: 36127 } } }
20:27:39.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:39.458925 3527 sys_catalog.cc:458] T 00000000000000000000000000000000 P b1731f8308c145f686f88efd6a54b851 [sys.catalog]: This master's current role is: FOLLOWER
20:27:39.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:39.460448 3543 catalog_manager.cc:1357] Generated new cluster ID: 3170e931b69f4506aaadd4d1dc1b2458
20:27:39.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:39.460589 3543 catalog_manager.cc:1505] Initializing Kudu internal certificate authority...
20:27:39.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:39.486260 3543 catalog_manager.cc:1380] Generated new certificate authority record
20:27:39.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:39.488099 3543 catalog_manager.cc:1514] Loading token signing keys...
20:27:39.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:39.501631 3424 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.46.60" port: 36127 } attempt: 5
20:27:39.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:39.513691 3543 catalog_manager.cc:6044] T 00000000000000000000000000000000 P 0db1d539946942f794b0c4ec81f8f4b4: Generated new TSK 0
20:27:39.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:39.515877 3543 catalog_manager.cc:1524] Initializing in-progress tserver states...
20:27:39.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:39.517458 3424 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P d6c270b387b04f17b2a58bfcb0eb4f82: Bootstrap starting.
20:27:39.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:39.526223 3256 external_mini_cluster.cc:1366] Running /tmp/dist-test-taska94UBC/build/asan/bin/kudu
20:27:39.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taska94UBC/build/asan/bin/kudu
20:27:39.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taska94UBC/test-tmp/mini-kudu-cluster1402564250707774898/ts-0/wal
20:27:39.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taska94UBC/test-tmp/mini-kudu-cluster1402564250707774898/ts-0/data
20:27:39.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:27:39.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:27:39.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:27:39.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:27:39.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:27:39.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:27:39.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taska94UBC/test-tmp/mini-kudu-cluster1402564250707774898/ts-0/logs
20:27:39.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taska94UBC/test-tmp/mini-kudu-cluster1402564250707774898/ts-0/data/info.pb
20:27:39.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:27:39.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:27:39.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:27:39.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:27:39.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:27:39.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:27:39.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:27:39.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:27:39.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
20:27:39.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:27:39.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.46.1:0
20:27:39.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.3.46.1
20:27:39.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.46.1
20:27:39.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:27:39.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127
20:27:39.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.46.20:45141
20:27:39.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:27:39.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:27:39.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:27:39.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
20:27:39.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
20:27:39.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true with env {}
20:27:39.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:39.535030 3424 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P d6c270b387b04f17b2a58bfcb0eb4f82: Neither blocks nor log segments found. Creating new log.
20:27:39.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:39.536724 3424 log.cc:826] T 00000000000000000000000000000000 P d6c270b387b04f17b2a58bfcb0eb4f82: Log is configured to *not* fsync() on all Append() calls
20:27:39.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:39.540831 3424 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P d6c270b387b04f17b2a58bfcb0eb4f82: No bootstrap required, opened a new log
20:27:39.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:39.549083 3424 raft_consensus.cc:359] T 00000000000000000000000000000000 P d6c270b387b04f17b2a58bfcb0eb4f82 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0db1d539946942f794b0c4ec81f8f4b4" member_type: VOTER last_known_addr { host: "127.3.46.62" port: 46071 } } peers { permanent_uuid: "d6c270b387b04f17b2a58bfcb0eb4f82" member_type: VOTER last_known_addr { host: "127.3.46.61" port: 39407 } } peers { permanent_uuid: "b1731f8308c145f686f88efd6a54b851" member_type: VOTER last_known_addr { host: "127.3.46.60" port: 36127 } }
20:27:39.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:39.549494 3424 raft_consensus.cc:385] T 00000000000000000000000000000000 P d6c270b387b04f17b2a58bfcb0eb4f82 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
20:27:39.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:39.549614 3424 raft_consensus.cc:740] T 00000000000000000000000000000000 P d6c270b387b04f17b2a58bfcb0eb4f82 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: d6c270b387b04f17b2a58bfcb0eb4f82, State: Initialized, Role: FOLLOWER
20:27:39.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:39.550541 3424 consensus_queue.cc:260] T 00000000000000000000000000000000 P d6c270b387b04f17b2a58bfcb0eb4f82 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0db1d539946942f794b0c4ec81f8f4b4" member_type: VOTER last_known_addr { host: "127.3.46.62" port: 46071 } } peers { permanent_uuid: "d6c270b387b04f17b2a58bfcb0eb4f82" member_type: VOTER last_known_addr { host: "127.3.46.61" port: 39407 } } peers { permanent_uuid: "b1731f8308c145f686f88efd6a54b851" member_type: VOTER last_known_addr { host: "127.3.46.60" port: 36127 } }
20:27:39.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:39.569979 3552 sys_catalog.cc:455] T 00000000000000000000000000000000 P d6c270b387b04f17b2a58bfcb0eb4f82 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0db1d539946942f794b0c4ec81f8f4b4" member_type: VOTER last_known_addr { host: "127.3.46.62" port: 46071 } } peers { permanent_uuid: "d6c270b387b04f17b2a58bfcb0eb4f82" member_type: VOTER last_known_addr { host: "127.3.46.61" port: 39407 } } peers { permanent_uuid: "b1731f8308c145f686f88efd6a54b851" member_type: VOTER last_known_addr { host: "127.3.46.60" port: 36127 } } }
20:27:39.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:39.570664 3552 sys_catalog.cc:458] T 00000000000000000000000000000000 P d6c270b387b04f17b2a58bfcb0eb4f82 [sys.catalog]: This master's current role is: FOLLOWER
20:27:39.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:39.571022 3552 raft_consensus.cc:493] T 00000000000000000000000000000000 P d6c270b387b04f17b2a58bfcb0eb4f82 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
20:27:39.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:39.571367 3552 raft_consensus.cc:515] T 00000000000000000000000000000000 P d6c270b387b04f17b2a58bfcb0eb4f82 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0db1d539946942f794b0c4ec81f8f4b4" member_type: VOTER last_known_addr { host: "127.3.46.62" port: 46071 } } peers { permanent_uuid: "d6c270b387b04f17b2a58bfcb0eb4f82" member_type: VOTER last_known_addr { host: "127.3.46.61" port: 39407 } } peers { permanent_uuid: "b1731f8308c145f686f88efd6a54b851" member_type: VOTER last_known_addr { host: "127.3.46.60" port: 36127 } }
20:27:39.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:39.573884 3308 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "d6c270b387b04f17b2a58bfcb0eb4f82" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "0db1d539946942f794b0c4ec81f8f4b4" is_pre_election: true
20:27:39.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:39.577224 3424 sys_catalog.cc:565] T 00000000000000000000000000000000 P d6c270b387b04f17b2a58bfcb0eb4f82 [sys.catalog]: configured and running, proceeding with master startup.
20:27:39.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:39.577651 3387 raft_consensus.cc:3060] T 00000000000000000000000000000000 P d6c270b387b04f17b2a58bfcb0eb4f82 [term 0 FOLLOWER]: Advancing to term 1
20:27:39.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:39.579926 3469 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "d6c270b387b04f17b2a58bfcb0eb4f82" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "b1731f8308c145f686f88efd6a54b851" is_pre_election: true
20:27:39.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:39.580742 3363 leader_election.cc:304] T 00000000000000000000000000000000 P d6c270b387b04f17b2a58bfcb0eb4f82 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: d6c270b387b04f17b2a58bfcb0eb4f82; no voters: 0db1d539946942f794b0c4ec81f8f4b4, b1731f8308c145f686f88efd6a54b851
20:27:39.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:39.581382 3387 raft_consensus.cc:1275] T 00000000000000000000000000000000 P d6c270b387b04f17b2a58bfcb0eb4f82 [term 1 FOLLOWER]: Refusing update from remote peer 0db1d539946942f794b0c4ec81f8f4b4: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
20:27:39.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:39.581384 3552 leader_election.cc:290] T 00000000000000000000000000000000 P d6c270b387b04f17b2a58bfcb0eb4f82 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 0db1d539946942f794b0c4ec81f8f4b4 (127.3.46.62:46071), b1731f8308c145f686f88efd6a54b851 (127.3.46.60:36127)
20:27:39.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:39.581849 3388 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "0db1d539946942f794b0c4ec81f8f4b4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d6c270b387b04f17b2a58bfcb0eb4f82"
20:27:39.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:39.582041 3551 raft_consensus.cc:2749] T 00000000000000000000000000000000 P d6c270b387b04f17b2a58bfcb0eb4f82 [term 1 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
20:27:39.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:39.582315 3389 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "0db1d539946942f794b0c4ec81f8f4b4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d6c270b387b04f17b2a58bfcb0eb4f82" is_pre_election: true
20:27:39.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:39.587707 3388 raft_consensus.cc:2468] T 00000000000000000000000000000000 P d6c270b387b04f17b2a58bfcb0eb4f82 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 0db1d539946942f794b0c4ec81f8f4b4 in term 1.
20:27:39.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:39.590629 3541 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 0db1d539946942f794b0c4ec81f8f4b4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d6c270b387b04f17b2a58bfcb0eb4f82" member_type: VOTER last_known_addr { host: "127.3.46.61" port: 39407 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
20:27:39.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:39.622566 3556 mvcc.cc:204] Tried to move back new op lower bound from 7281325094550441984 to 7281325094439702528. Current Snapshot: MvccSnapshot[applied={T|T < 7281325094550441984}]
20:27:39.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:39.630242 3552 sys_catalog.cc:455] T 00000000000000000000000000000000 P d6c270b387b04f17b2a58bfcb0eb4f82 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "0db1d539946942f794b0c4ec81f8f4b4" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0db1d539946942f794b0c4ec81f8f4b4" member_type: VOTER last_known_addr { host: "127.3.46.62" port: 46071 } } peers { permanent_uuid: "d6c270b387b04f17b2a58bfcb0eb4f82" member_type: VOTER last_known_addr { host: "127.3.46.61" port: 39407 } } peers { permanent_uuid: "b1731f8308c145f686f88efd6a54b851" member_type: VOTER last_known_addr { host: "127.3.46.60" port: 36127 } } }
20:27:39.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:39.630288 3551 sys_catalog.cc:455] T 00000000000000000000000000000000 P d6c270b387b04f17b2a58bfcb0eb4f82 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 0db1d539946942f794b0c4ec81f8f4b4. Latest consensus state: current_term: 1 leader_uuid: "0db1d539946942f794b0c4ec81f8f4b4" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0db1d539946942f794b0c4ec81f8f4b4" member_type: VOTER last_known_addr { host: "127.3.46.62" port: 46071 } } peers { permanent_uuid: "d6c270b387b04f17b2a58bfcb0eb4f82" member_type: VOTER last_known_addr { host: "127.3.46.61" port: 39407 } } peers { permanent_uuid: "b1731f8308c145f686f88efd6a54b851" member_type: VOTER last_known_addr { host: "127.3.46.60" port: 36127 } } }
20:27:39.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:39.630683 3552 sys_catalog.cc:458] T 00000000000000000000000000000000 P d6c270b387b04f17b2a58bfcb0eb4f82 [sys.catalog]: This master's current role is: FOLLOWER
20:27:39.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:39.630726 3551 sys_catalog.cc:458] T 00000000000000000000000000000000 P d6c270b387b04f17b2a58bfcb0eb4f82 [sys.catalog]: This master's current role is: FOLLOWER
20:27:39.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:39.654481 3567 catalog_manager.cc:805] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
20:27:39.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:39.674640 3541 sys_catalog.cc:455] T 00000000000000000000000000000000 P 0db1d539946942f794b0c4ec81f8f4b4 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "0db1d539946942f794b0c4ec81f8f4b4" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0db1d539946942f794b0c4ec81f8f4b4" member_type: VOTER last_known_addr { host: "127.3.46.62" port: 46071 } } peers { permanent_uuid: "d6c270b387b04f17b2a58bfcb0eb4f82" member_type: VOTER last_known_addr { host: "127.3.46.61" port: 39407 } } peers { permanent_uuid: "b1731f8308c145f686f88efd6a54b851" member_type: VOTER last_known_addr { host: "127.3.46.60" port: 36127 } } }
20:27:39.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:39.674932 3541 sys_catalog.cc:458] T 00000000000000000000000000000000 P 0db1d539946942f794b0c4ec81f8f4b4 [sys.catalog]: This master's current role is: LEADER
20:27:40.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:27:40.039770 3549 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
20:27:40.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:27:40.040096 3549 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:27:40.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:27:40.040186 3549 flags.cc:432] Enabled unsafe flag: --never_fsync=true
20:27:40.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:27:40.049055 3549 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
20:27:40.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:27:40.049201 3549 flags.cc:432] Enabled experimental flag: --txn_keepalive_interval_ms=1000
20:27:40.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:27:40.049252 3549 flags.cc:432] Enabled experimental flag: --txn_staleness_tracker_interval_ms=250
20:27:40.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:27:40.049288 3549 flags.cc:432] Enabled experimental flag: --enable_txn_system_client_init=true
20:27:40.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:27:40.049366 3549 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.3.46.1
20:27:40.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:40.060570 3549 tablet_server_runner.cc:78] Tablet server non-default flags:
20:27:40.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:27:40.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.46.20:45141
20:27:40.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:27:40.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:27:40.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taska94UBC/test-tmp/mini-kudu-cluster1402564250707774898/ts-0/data
20:27:40.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taska94UBC/test-tmp/mini-kudu-cluster1402564250707774898/ts-0/wal
20:27:40.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:27:40.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:27:40.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.46.1:0
20:27:40.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:27:40.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:27:40.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:27:40.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taska94UBC/test-tmp/mini-kudu-cluster1402564250707774898/ts-0/data/info.pb
20:27:40.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.46.1
20:27:40.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:27:40.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
20:27:40.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
20:27:40.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true
20:27:40.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127
20:27:40.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:27:40.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:27:40.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:27:40.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:27:40.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:27:40.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.3.46.1
20:27:40.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taska94UBC/test-tmp/mini-kudu-cluster1402564250707774898/ts-0/logs
20:27:40.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:27:40.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:27:40.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:27:40.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
20:27:40.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
20:27:40.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0ba7f54e15c9f673bf3e350f30b8d5f1a4037b79
20:27:40.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:27:40.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 01 May 2026 19:49:29 UTC on bdcb31816ec0
20:27:40.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11698
20:27:40.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
20:27:40.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:40.062382 3549 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:27:40.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:40.064440 3549 file_cache.cc:492] Constructed file cache file cache with capacity 419430
20:27:40.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:27:40.075551 3573 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:27:40.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:27:40.076169 3572 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:27:40.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:27:40.076371 3575 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:27:40.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:40.076782 3549 server_base.cc:1061] running on GCE node
20:27:40.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:40.077566 3549 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:27:40.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:40.078766 3549 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:27:40.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:40.080183 3549 hybrid_clock.cc:648] HybridClock initialized: now 1777667260080023 us; error 67 us; skew 500 ppm
20:27:40.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:40.080571 3549 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:27:40.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:40.083681 3549 webserver.cc:492] Webserver started at http://127.3.46.1:35431/ using document root <none> and password file <none>
20:27:40.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:40.084564 3549 fs_manager.cc:362] Metadata directory not provided
20:27:40.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:40.084714 3549 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:27:40.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:40.085095 3549 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
20:27:40.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:40.087795 3549 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taska94UBC/test-tmp/mini-kudu-cluster1402564250707774898/ts-0/data/instance:
20:27:40.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d00c04330aef4a87a22435384a1a5c12"
20:27:40.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-01 20:27:40 on dist-test-slave-f471"
20:27:40.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:40.088653 3549 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taska94UBC/test-tmp/mini-kudu-cluster1402564250707774898/ts-0/wal/instance:
20:27:40.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d00c04330aef4a87a22435384a1a5c12"
20:27:40.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-01 20:27:40 on dist-test-slave-f471"
20:27:40.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:40.094743 3549 fs_manager.cc:696] Time spent creating directory manager: real 0.006s user 0.007s sys 0.001s
20:27:40.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:40.099849 3581 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:27:40.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:40.101679 3549 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.005s sys 0.000s
20:27:40.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:40.101972 3549 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taska94UBC/test-tmp/mini-kudu-cluster1402564250707774898/ts-0/data,/tmp/dist-test-taska94UBC/test-tmp/mini-kudu-cluster1402564250707774898/ts-0/wal
20:27:40.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d00c04330aef4a87a22435384a1a5c12"
20:27:40.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-01 20:27:40 on dist-test-slave-f471"
20:27:40.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:40.102219 3549 fs_report.cc:389] FS layout report
20:27:40.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:27:40.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taska94UBC/test-tmp/mini-kudu-cluster1402564250707774898/ts-0/wal
20:27:40.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taska94UBC/test-tmp/mini-kudu-cluster1402564250707774898/ts-0/wal
20:27:40.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taska94UBC/test-tmp/mini-kudu-cluster1402564250707774898/ts-0/data/data
20:27:40.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:27:40.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:27:40.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:27:40.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:27:40.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:27:40.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:27:40.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:27:40.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:27:40.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:27:40.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:27:40.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:27:40.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:40.148152 3549 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
20:27:40.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:40.149104 3549 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:27:40.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:40.149425 3549 kserver.cc:163] Server-wide thread pool size limit: 3276
20:27:40.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:40.152585 3549 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
20:27:40.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:40.152717 3549 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
20:27:40.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:40.152837 3549 ts_tablet_manager.cc:616] Registered 0 tablets
20:27:40.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:40.152915 3549 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
20:27:40.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:40.216325 3549 rpc_server.cc:307] RPC server started. Bound to: 127.3.46.1:37067
20:27:40.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:40.216380 3705 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.3.46.1:37067 every 8 connection(s)
20:27:40.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:40.217679 3549 server_base.cc:1193] Dumped server information to /tmp/dist-test-taska94UBC/test-tmp/mini-kudu-cluster1402564250707774898/ts-0/data/info.pb
20:27:40.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:40.218191 3256 external_mini_cluster.cc:1428] Started /tmp/dist-test-taska94UBC/build/asan/bin/kudu as pid 3549
20:27:40.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:40.218402 3256 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taska94UBC/test-tmp/mini-kudu-cluster1402564250707774898/ts-0/wal/instance
20:27:40.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:40.224216 3707 heartbeater.cc:344] Connected to a master server at 127.3.46.62:46071
20:27:40.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:40.224627 3707 heartbeater.cc:461] Registering TS with master...
20:27:40.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:40.224670 3256 external_mini_cluster.cc:1366] Running /tmp/dist-test-taska94UBC/build/asan/bin/kudu
20:27:40.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taska94UBC/build/asan/bin/kudu
20:27:40.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taska94UBC/test-tmp/mini-kudu-cluster1402564250707774898/ts-1/wal
20:27:40.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taska94UBC/test-tmp/mini-kudu-cluster1402564250707774898/ts-1/data
20:27:40.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:27:40.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:27:40.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:27:40.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:27:40.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:27:40.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:27:40.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taska94UBC/test-tmp/mini-kudu-cluster1402564250707774898/ts-1/logs
20:27:40.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taska94UBC/test-tmp/mini-kudu-cluster1402564250707774898/ts-1/data/info.pb
20:27:40.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:27:40.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:27:40.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:27:40.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:27:40.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:27:40.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:27:40.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:27:40.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:27:40.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
20:27:40.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:27:40.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.46.2:0
20:27:40.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.3.46.2
20:27:40.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.46.2
20:27:40.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:27:40.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127
20:27:40.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.46.20:45141
20:27:40.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:27:40.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:27:40.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:27:40.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
20:27:40.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
20:27:40.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true with env {}
20:27:40.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:40.225741 3707 heartbeater.cc:507] Master 127.3.46.62:46071 requested a full tablet report, sending...
20:27:40.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:40.228139 3297 ts_manager.cc:194] Registered new tserver with Master: d00c04330aef4a87a22435384a1a5c12 (127.3.46.1:37067)
20:27:40.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:40.229116 3706 heartbeater.cc:344] Connected to a master server at 127.3.46.60:36127
20:27:40.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:40.229267 3706 heartbeater.cc:461] Registering TS with master...
20:27:40.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:40.229866 3706 heartbeater.cc:507] Master 127.3.46.60:36127 requested a full tablet report, sending...
20:27:40.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:40.230403 3297 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.3.46.1:51745
20:27:40.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:40.231827 3460 ts_manager.cc:194] Registered new tserver with Master: d00c04330aef4a87a22435384a1a5c12 (127.3.46.1:37067)
20:27:40.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:40.232534 3708 heartbeater.cc:344] Connected to a master server at 127.3.46.61:39407
20:27:40.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:40.232687 3708 heartbeater.cc:461] Registering TS with master...
20:27:40.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:40.233027 3708 heartbeater.cc:507] Master 127.3.46.61:39407 requested a full tablet report, sending...
20:27:40.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:40.234694 3378 ts_manager.cc:194] Registered new tserver with Master: d00c04330aef4a87a22435384a1a5c12 (127.3.46.1:37067)
20:27:40.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:40.359954 3539 catalog_manager.cc:1269] Loaded cluster ID: 3170e931b69f4506aaadd4d1dc1b2458
20:27:40.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:40.360066 3539 catalog_manager.cc:1562] T 00000000000000000000000000000000 P b1731f8308c145f686f88efd6a54b851: loading cluster ID for follower catalog manager: success
20:27:40.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:40.363849 3539 catalog_manager.cc:1584] T 00000000000000000000000000000000 P b1731f8308c145f686f88efd6a54b851: acquiring CA information for follower catalog manager: success
20:27:40.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:40.366323 3539 catalog_manager.cc:1612] T 00000000000000000000000000000000 P b1731f8308c145f686f88efd6a54b851: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
20:27:40.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:27:40.560084 3711 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
20:27:40.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:27:40.560417 3711 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:27:40.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:27:40.560544 3711 flags.cc:432] Enabled unsafe flag: --never_fsync=true
20:27:40.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:27:40.569247 3711 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
20:27:40.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:27:40.569413 3711 flags.cc:432] Enabled experimental flag: --txn_keepalive_interval_ms=1000
20:27:40.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:27:40.569482 3711 flags.cc:432] Enabled experimental flag: --txn_staleness_tracker_interval_ms=250
20:27:40.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:27:40.569530 3711 flags.cc:432] Enabled experimental flag: --enable_txn_system_client_init=true
20:27:40.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:27:40.569650 3711 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.3.46.2
20:27:40.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:40.580031 3711 tablet_server_runner.cc:78] Tablet server non-default flags:
20:27:40.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:27:40.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.46.20:45141
20:27:40.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:27:40.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:27:40.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taska94UBC/test-tmp/mini-kudu-cluster1402564250707774898/ts-1/data
20:27:40.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taska94UBC/test-tmp/mini-kudu-cluster1402564250707774898/ts-1/wal
20:27:40.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:27:40.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:27:40.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.46.2:0
20:27:40.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:27:40.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:27:40.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:27:40.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taska94UBC/test-tmp/mini-kudu-cluster1402564250707774898/ts-1/data/info.pb
20:27:40.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.46.2
20:27:40.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:27:40.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
20:27:40.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
20:27:40.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true
20:27:40.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127
20:27:40.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:27:40.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:27:40.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:27:40.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:27:40.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:27:40.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.3.46.2
20:27:40.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taska94UBC/test-tmp/mini-kudu-cluster1402564250707774898/ts-1/logs
20:27:40.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:27:40.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:27:40.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:27:40.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
20:27:40.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
20:27:40.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0ba7f54e15c9f673bf3e350f30b8d5f1a4037b79
20:27:40.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:27:40.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 01 May 2026 19:49:29 UTC on bdcb31816ec0
20:27:40.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11698
20:27:40.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
20:27:40.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:40.582086 3711 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:27:40.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:40.583904 3711 file_cache.cc:492] Constructed file cache file cache with capacity 419430
20:27:40.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:27:40.593865 3717 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:27:40.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:27:40.593925 3719 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:27:40.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:27:40.594252 3716 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:27:40.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:40.594827 3711 server_base.cc:1061] running on GCE node
20:27:40.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:40.595485 3711 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:27:40.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:40.596614 3711 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:27:40.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:40.597859 3711 hybrid_clock.cc:648] HybridClock initialized: now 1777667260597788 us; error 44 us; skew 500 ppm
20:27:40.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:40.598212 3711 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:27:40.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:40.600838 3711 webserver.cc:492] Webserver started at http://127.3.46.2:36313/ using document root <none> and password file <none>
20:27:40.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:40.601756 3711 fs_manager.cc:362] Metadata directory not provided
20:27:40.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:40.601903 3711 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:27:40.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:40.602254 3711 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
20:27:40.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:40.604753 3711 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taska94UBC/test-tmp/mini-kudu-cluster1402564250707774898/ts-1/data/instance:
20:27:40.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "40c77f37a81a464ba7ff42f17bd32c06"
20:27:40.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-01 20:27:40 on dist-test-slave-f471"
20:27:40.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:40.605571 3711 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taska94UBC/test-tmp/mini-kudu-cluster1402564250707774898/ts-1/wal/instance:
20:27:40.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "40c77f37a81a464ba7ff42f17bd32c06"
20:27:40.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-01 20:27:40 on dist-test-slave-f471"
20:27:40.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:40.611445 3711 fs_manager.cc:696] Time spent creating directory manager: real 0.005s user 0.004s sys 0.000s
20:27:40.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:40.615151 3725 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:27:40.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:40.616765 3711 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.004s sys 0.000s
20:27:40.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:40.616991 3711 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taska94UBC/test-tmp/mini-kudu-cluster1402564250707774898/ts-1/data,/tmp/dist-test-taska94UBC/test-tmp/mini-kudu-cluster1402564250707774898/ts-1/wal
20:27:40.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "40c77f37a81a464ba7ff42f17bd32c06"
20:27:40.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-01 20:27:40 on dist-test-slave-f471"
20:27:40.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:40.617221 3711 fs_report.cc:389] FS layout report
20:27:40.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:27:40.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taska94UBC/test-tmp/mini-kudu-cluster1402564250707774898/ts-1/wal
20:27:40.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taska94UBC/test-tmp/mini-kudu-cluster1402564250707774898/ts-1/wal
20:27:40.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taska94UBC/test-tmp/mini-kudu-cluster1402564250707774898/ts-1/data/data
20:27:40.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:27:40.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:27:40.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:27:40.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:27:40.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:27:40.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:27:40.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:27:40.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:27:40.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:27:40.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:27:40.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:27:40.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:40.660377 3567 catalog_manager.cc:1269] Loaded cluster ID: 3170e931b69f4506aaadd4d1dc1b2458
20:27:40.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:40.660521 3567 catalog_manager.cc:1562] T 00000000000000000000000000000000 P d6c270b387b04f17b2a58bfcb0eb4f82: loading cluster ID for follower catalog manager: success
20:27:40.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:40.664776 3567 catalog_manager.cc:1584] T 00000000000000000000000000000000 P d6c270b387b04f17b2a58bfcb0eb4f82: acquiring CA information for follower catalog manager: success
20:27:40.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:40.667008 3711 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
20:27:40.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:40.667595 3567 catalog_manager.cc:1612] T 00000000000000000000000000000000 P d6c270b387b04f17b2a58bfcb0eb4f82: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
20:27:40.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:40.668015 3711 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:27:40.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:40.668318 3711 kserver.cc:163] Server-wide thread pool size limit: 3276
20:27:40.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:40.671855 3711 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
20:27:40.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:40.672153 3711 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.001s sys 0.000s
20:27:40.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:40.672379 3711 ts_tablet_manager.cc:616] Registered 0 tablets
20:27:40.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:40.672468 3711 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
20:27:40.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:40.749854 3711 rpc_server.cc:307] RPC server started. Bound to: 127.3.46.2:37949
20:27:40.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:40.749889 3847 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.3.46.2:37949 every 8 connection(s)
20:27:40.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:40.751324 3711 server_base.cc:1193] Dumped server information to /tmp/dist-test-taska94UBC/test-tmp/mini-kudu-cluster1402564250707774898/ts-1/data/info.pb
20:27:40.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:40.753968 3849 heartbeater.cc:344] Connected to a master server at 127.3.46.62:46071
20:27:40.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:40.758158 3256 external_mini_cluster.cc:1428] Started /tmp/dist-test-taska94UBC/build/asan/bin/kudu as pid 3711
20:27:40.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:40.758426 3256 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taska94UBC/test-tmp/mini-kudu-cluster1402564250707774898/ts-1/wal/instance
20:27:40.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:40.758812 3849 heartbeater.cc:461] Registering TS with master...
20:27:40.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:40.760150 3849 heartbeater.cc:507] Master 127.3.46.62:46071 requested a full tablet report, sending...
20:27:40.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:40.765095 3256 external_mini_cluster.cc:1366] Running /tmp/dist-test-taska94UBC/build/asan/bin/kudu
20:27:40.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taska94UBC/build/asan/bin/kudu
20:27:40.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taska94UBC/test-tmp/mini-kudu-cluster1402564250707774898/ts-2/wal
20:27:40.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taska94UBC/test-tmp/mini-kudu-cluster1402564250707774898/ts-2/data
20:27:40.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:27:40.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:27:40.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:27:40.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:27:40.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:27:40.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:27:40.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taska94UBC/test-tmp/mini-kudu-cluster1402564250707774898/ts-2/logs
20:27:40.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taska94UBC/test-tmp/mini-kudu-cluster1402564250707774898/ts-2/data/info.pb
20:27:40.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:27:40.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:27:40.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:27:40.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:27:40.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:27:40.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:27:40.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:27:40.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:27:40.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
20:27:40.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:27:40.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.46.3:0
20:27:40.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.3.46.3
20:27:40.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.46.3
20:27:40.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:27:40.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127
20:27:40.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.46.20:45141
20:27:40.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:27:40.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:27:40.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:27:40.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
20:27:40.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
20:27:40.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true with env {}
20:27:40.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:40.772027 3848 heartbeater.cc:344] Connected to a master server at 127.3.46.60:36127
20:27:40.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:40.772428 3848 heartbeater.cc:461] Registering TS with master...
20:27:40.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:40.773149 3848 heartbeater.cc:507] Master 127.3.46.60:36127 requested a full tablet report, sending...
20:27:40.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:40.774494 3460 ts_manager.cc:194] Registered new tserver with Master: 40c77f37a81a464ba7ff42f17bd32c06 (127.3.46.2:37949)
20:27:40.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:40.778474 3297 ts_manager.cc:194] Registered new tserver with Master: 40c77f37a81a464ba7ff42f17bd32c06 (127.3.46.2:37949)
20:27:40.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:40.779709 3297 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.3.46.2:59061
20:27:40.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:40.782339 3850 heartbeater.cc:344] Connected to a master server at 127.3.46.61:39407
20:27:40.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:40.782541 3850 heartbeater.cc:461] Registering TS with master...
20:27:40.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:40.783031 3850 heartbeater.cc:507] Master 127.3.46.61:39407 requested a full tablet report, sending...
20:27:40.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:40.784597 3378 ts_manager.cc:194] Registered new tserver with Master: 40c77f37a81a464ba7ff42f17bd32c06 (127.3.46.2:37949)
20:27:41.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:27:41.143271 3853 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
20:27:41.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:27:41.143563 3853 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:27:41.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:27:41.143647 3853 flags.cc:432] Enabled unsafe flag: --never_fsync=true
20:27:41.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:27:41.152704 3853 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
20:27:41.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:27:41.152848 3853 flags.cc:432] Enabled experimental flag: --txn_keepalive_interval_ms=1000
20:27:41.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:27:41.152895 3853 flags.cc:432] Enabled experimental flag: --txn_staleness_tracker_interval_ms=250
20:27:41.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:27:41.152932 3853 flags.cc:432] Enabled experimental flag: --enable_txn_system_client_init=true
20:27:41.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:27:41.153012 3853 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.3.46.3
20:27:41.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:41.163456 3853 tablet_server_runner.cc:78] Tablet server non-default flags:
20:27:41.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:27:41.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.46.20:45141
20:27:41.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:27:41.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:27:41.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taska94UBC/test-tmp/mini-kudu-cluster1402564250707774898/ts-2/data
20:27:41.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taska94UBC/test-tmp/mini-kudu-cluster1402564250707774898/ts-2/wal
20:27:41.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:27:41.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:27:41.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.46.3:0
20:27:41.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:27:41.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:27:41.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:27:41.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taska94UBC/test-tmp/mini-kudu-cluster1402564250707774898/ts-2/data/info.pb
20:27:41.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.46.3
20:27:41.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:27:41.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
20:27:41.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
20:27:41.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true
20:27:41.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127
20:27:41.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:27:41.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:27:41.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:27:41.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:27:41.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:27:41.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.3.46.3
20:27:41.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taska94UBC/test-tmp/mini-kudu-cluster1402564250707774898/ts-2/logs
20:27:41.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:27:41.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:27:41.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:27:41.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
20:27:41.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
20:27:41.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0ba7f54e15c9f673bf3e350f30b8d5f1a4037b79
20:27:41.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:27:41.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 01 May 2026 19:49:29 UTC on bdcb31816ec0
20:27:41.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11698
20:27:41.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
20:27:41.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:41.164999 3853 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:27:41.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:41.166877 3853 file_cache.cc:492] Constructed file cache file cache with capacity 419430
20:27:41.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:27:41.177003 3858 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:27:41.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:27:41.176925 3859 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:27:41.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:27:41.177166 3861 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:27:41.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:41.178200 3853 server_base.cc:1061] running on GCE node
20:27:41.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:41.178848 3853 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:27:41.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:41.179795 3853 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:27:41.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:41.181075 3853 hybrid_clock.cc:648] HybridClock initialized: now 1777667261180982 us; error 48 us; skew 500 ppm
20:27:41.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:41.181414 3853 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:27:41.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:41.184020 3853 webserver.cc:492] Webserver started at http://127.3.46.3:37265/ using document root <none> and password file <none>
20:27:41.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:41.184731 3853 fs_manager.cc:362] Metadata directory not provided
20:27:41.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:41.184841 3853 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:27:41.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:41.185138 3853 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
20:27:41.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:41.188191 3853 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taska94UBC/test-tmp/mini-kudu-cluster1402564250707774898/ts-2/data/instance:
20:27:41.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e149de1a3f47443487db92e7c70143dd"
20:27:41.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-01 20:27:41 on dist-test-slave-f471"
20:27:41.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:41.189157 3853 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taska94UBC/test-tmp/mini-kudu-cluster1402564250707774898/ts-2/wal/instance:
20:27:41.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e149de1a3f47443487db92e7c70143dd"
20:27:41.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-01 20:27:41 on dist-test-slave-f471"
20:27:41.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:41.195026 3853 fs_manager.cc:696] Time spent creating directory manager: real 0.005s user 0.007s sys 0.001s
20:27:41.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:41.199331 3867 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:27:41.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:41.201324 3853 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.003s sys 0.001s
20:27:41.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:41.201515 3853 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taska94UBC/test-tmp/mini-kudu-cluster1402564250707774898/ts-2/data,/tmp/dist-test-taska94UBC/test-tmp/mini-kudu-cluster1402564250707774898/ts-2/wal
20:27:41.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e149de1a3f47443487db92e7c70143dd"
20:27:41.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-01 20:27:41 on dist-test-slave-f471"
20:27:41.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:41.201709 3853 fs_report.cc:389] FS layout report
20:27:41.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:27:41.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taska94UBC/test-tmp/mini-kudu-cluster1402564250707774898/ts-2/wal
20:27:41.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taska94UBC/test-tmp/mini-kudu-cluster1402564250707774898/ts-2/wal
20:27:41.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taska94UBC/test-tmp/mini-kudu-cluster1402564250707774898/ts-2/data/data
20:27:41.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:27:41.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:27:41.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:27:41.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:27:41.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:27:41.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:27:41.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:27:41.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:27:41.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:27:41.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:27:41.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:27:41.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:41.233582 3707 heartbeater.cc:499] Master 127.3.46.62:46071 was elected leader, sending a full tablet report...
20:27:41.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:41.243398 3853 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
20:27:41.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:41.244382 3853 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:27:41.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:41.244634 3853 kserver.cc:163] Server-wide thread pool size limit: 3276
20:27:41.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:41.247777 3853 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
20:27:41.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:41.247884 3853 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
20:27:41.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:41.247998 3853 ts_tablet_manager.cc:616] Registered 0 tablets
20:27:41.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:41.248075 3853 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
20:27:41.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:41.312237 3853 rpc_server.cc:307] RPC server started. Bound to: 127.3.46.3:42083
20:27:41.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:41.312281 3989 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.3.46.3:42083 every 8 connection(s)
20:27:41.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:41.313669 3853 server_base.cc:1193] Dumped server information to /tmp/dist-test-taska94UBC/test-tmp/mini-kudu-cluster1402564250707774898/ts-2/data/info.pb
20:27:41.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:41.316215 3256 external_mini_cluster.cc:1428] Started /tmp/dist-test-taska94UBC/build/asan/bin/kudu as pid 3853
20:27:41.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:41.316394 3256 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taska94UBC/test-tmp/mini-kudu-cluster1402564250707774898/ts-2/wal/instance
20:27:41.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:41.317233 3991 heartbeater.cc:344] Connected to a master server at 127.3.46.62:46071
20:27:41.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:41.317795 3991 heartbeater.cc:461] Registering TS with master...
20:27:41.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:41.319144 3991 heartbeater.cc:507] Master 127.3.46.62:46071 requested a full tablet report, sending...
20:27:41.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:41.320955 3297 ts_manager.cc:194] Registered new tserver with Master: e149de1a3f47443487db92e7c70143dd (127.3.46.3:42083)
20:27:41.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:41.322041 3297 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.3.46.3:33401
20:27:41.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:41.325551 3992 heartbeater.cc:344] Connected to a master server at 127.3.46.61:39407
20:27:41.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:41.325889 3992 heartbeater.cc:461] Registering TS with master...
20:27:41.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:41.326395 3992 heartbeater.cc:507] Master 127.3.46.61:39407 requested a full tablet report, sending...
20:27:41.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:41.326632 3990 heartbeater.cc:344] Connected to a master server at 127.3.46.60:36127
20:27:41.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:41.326751 3990 heartbeater.cc:461] Registering TS with master...
20:27:41.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:41.327039 3990 heartbeater.cc:507] Master 127.3.46.60:36127 requested a full tablet report, sending...
20:27:41.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:41.327739 3378 ts_manager.cc:194] Registered new tserver with Master: e149de1a3f47443487db92e7c70143dd (127.3.46.3:42083)
20:27:41.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:41.328116 3460 ts_manager.cc:194] Registered new tserver with Master: e149de1a3f47443487db92e7c70143dd (127.3.46.3:42083)
20:27:41.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:41.342825 3256 external_mini_cluster.cc:949] 3 TS(s) registered with all masters
20:27:41.344 [DEBUG - main] (MiniKuduCluster.java:195) Response:
20:27:41.345 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
20:27:41.346 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.3.46.62"
port: 46071
}
bound_http_address {
host: "127.3.46.62"
port: 33105
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.3.46.61"
port: 39407
}
bound_http_address {
host: "127.3.46.61"
port: 45983
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.3.46.60"
port: 36127
}
bound_http_address {
host: "127.3.46.60"
port: 38161
}
}
}
20:27:41.346 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
20:27:41.347 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.3.46.1"
port: 37067
}
bound_http_address {
host: "127.3.46.1"
port: 35431
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.3.46.2"
port: 37949
}
bound_http_address {
host: "127.3.46.2"
port: 36313
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.3.46.3"
port: 42083
}
bound_http_address {
host: "127.3.46.3"
port: 37265
}
}
}
20:27:41.347 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
20:27:41.348 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
20:27:41.350 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.3.46.62:46071(127.3.46.62:46071)] connecting to peer
20:27:41.351 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:41.352 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.3.46.62:46071(127.3.46.62:46071)] Successfully connected to peer
20:27:41.353 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.3.46.61:39407(127.3.46.61:39407)] Successfully connected to peer
20:27:41.353 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:41.354 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.3.46.62
20:27:41.354 [DEBUG - kudu-nio-1] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.3.46.61
20:27:41.355 [DEBUG - kudu-nio-2] (Connection.java:757) [peer master-127.3.46.60:36127(127.3.46.60:36127)] Successfully connected to peer
20:27:41.356 [DEBUG - kudu-nio-2] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.3.46.60
20:27:41.368 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x4a49d5ba, L:/127.0.0.1:47554 - R:/127.3.46.62:46071] using SASL/PLAIN
20:27:41.368 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0x01435035, L:/127.0.0.1:44598 - R:/127.3.46.61:39407] using SASL/PLAIN
20:27:41.371 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
20:27:41.371 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=CreateTable, tablet=null, attempt=1, TimeoutTracker(timeout=50000, elapsed=21), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [3ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [20ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: OK, [20ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, deferred=null) after lookup
20:27:41.376 [DEBUG - kudu-nio-2] (Negotiator.java:919) Authenticated connection [id: 0x8609ca1e, L:/127.0.0.1:54942 - R:/127.3.46.60:36127] using SASL/PLAIN
20:27:41.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:41.372916 3297 catalog_manager.cc:2257] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47554:
20:27:41.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_manager_fallback_rolling"
20:27:41.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
20:27:41.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
20:27:41.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "key"
20:27:41.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT32
20:27:41.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
20:27:41.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
20:27:41.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
20:27:41.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
20:27:41.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
20:27:41.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
20:27:41.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
20:27:41.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "column1_i"
20:27:41.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT32
20:27:41.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
20:27:41.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
20:27:41.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
20:27:41.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
20:27:41.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
20:27:41.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
20:27:41.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
20:27:41.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "column2_i"
20:27:41.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT32
20:27:41.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
20:27:41.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
20:27:41.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
20:27:41.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
20:27:41.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
20:27:41.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
20:27:41.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
20:27:41.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "column3_s"
20:27:41.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
20:27:41.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
20:27:41.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: true
20:27:41.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: DICT_ENCODING
20:27:41.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: LZ4
20:27:41.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 4096
20:27:41.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
20:27:41.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
20:27:41.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
20:27:41.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
20:27:41.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "column4_b"
20:27:41.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: BOOL
20:27:41.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
20:27:41.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
20:27:41.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
20:27:41.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
20:27:41.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
20:27:41.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
20:27:41.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
20:27:41.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
20:27:41.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) hash_schema {
20:27:41.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
20:27:41.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "key"
20:27:41.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
20:27:41.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_buckets: 2
20:27:41.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) seed: 0
20:27:41.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
20:27:41.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
20:27:41.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:27:41.375344 3297 catalog_manager.cc:7033] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table txn_manager_fallback_rolling in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
20:27:41.415 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:894) Opened table 346e6e6149384e3992393d04f9f16b9c
20:27:41.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:41.451614 3779 tablet_service.cc:1511] Processing CreateTablet for tablet f9ccfdb32e164e61bde68775baab7f9d (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=346e6e6149384e3992393d04f9f16b9c]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
20:27:41.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:41.452739 3778 tablet_service.cc:1511] Processing CreateTablet for tablet fe829cba01eb4acb9847b261804f81c8 (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=346e6e6149384e3992393d04f9f16b9c]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
20:27:41.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:41.453555 3779 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet f9ccfdb32e164e61bde68775baab7f9d. 1 dirs total, 0 dirs full, 0 dirs failed
20:27:41.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:41.454196 3778 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet fe829cba01eb4acb9847b261804f81c8. 1 dirs total, 0 dirs full, 0 dirs failed
20:27:41.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:41.459676 3640 tablet_service.cc:1511] Processing CreateTablet for tablet fe829cba01eb4acb9847b261804f81c8 (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=346e6e6149384e3992393d04f9f16b9c]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
20:27:41.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:41.459463 3641 tablet_service.cc:1511] Processing CreateTablet for tablet f9ccfdb32e164e61bde68775baab7f9d (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=346e6e6149384e3992393d04f9f16b9c]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
20:27:41.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:41.461081 3640 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet fe829cba01eb4acb9847b261804f81c8. 1 dirs total, 0 dirs full, 0 dirs failed
20:27:41.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:41.461555 3641 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet f9ccfdb32e164e61bde68775baab7f9d. 1 dirs total, 0 dirs full, 0 dirs failed
20:27:41.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:41.461103 3924 tablet_service.cc:1511] Processing CreateTablet for tablet fe829cba01eb4acb9847b261804f81c8 (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=346e6e6149384e3992393d04f9f16b9c]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
20:27:41.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:41.462656 3924 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet fe829cba01eb4acb9847b261804f81c8. 1 dirs total, 0 dirs full, 0 dirs failed
20:27:41.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:41.463066 3925 tablet_service.cc:1511] Processing CreateTablet for tablet f9ccfdb32e164e61bde68775baab7f9d (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=346e6e6149384e3992393d04f9f16b9c]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
20:27:41.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:41.463574 3925 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet f9ccfdb32e164e61bde68775baab7f9d. 1 dirs total, 0 dirs full, 0 dirs failed
20:27:41.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:41.477999 4017 tablet_bootstrap.cc:492] T f9ccfdb32e164e61bde68775baab7f9d P 40c77f37a81a464ba7ff42f17bd32c06: Bootstrap starting.
20:27:41.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:41.480988 4017 tablet_bootstrap.cc:654] T f9ccfdb32e164e61bde68775baab7f9d P 40c77f37a81a464ba7ff42f17bd32c06: Neither blocks nor log segments found. Creating new log.
20:27:41.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:41.482218 4018 tablet_bootstrap.cc:492] T f9ccfdb32e164e61bde68775baab7f9d P d00c04330aef4a87a22435384a1a5c12: Bootstrap starting.
20:27:41.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:41.482206 4017 log.cc:826] T f9ccfdb32e164e61bde68775baab7f9d P 40c77f37a81a464ba7ff42f17bd32c06: Log is configured to *not* fsync() on all Append() calls
20:27:41.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:41.486864 4018 tablet_bootstrap.cc:654] T f9ccfdb32e164e61bde68775baab7f9d P d00c04330aef4a87a22435384a1a5c12: Neither blocks nor log segments found. Creating new log.
20:27:41.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:41.488423 4018 log.cc:826] T f9ccfdb32e164e61bde68775baab7f9d P d00c04330aef4a87a22435384a1a5c12: Log is configured to *not* fsync() on all Append() calls
20:27:41.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:41.489670 4020 tablet_bootstrap.cc:492] T fe829cba01eb4acb9847b261804f81c8 P e149de1a3f47443487db92e7c70143dd: Bootstrap starting.
20:27:41.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:41.491653 4017 tablet_bootstrap.cc:492] T f9ccfdb32e164e61bde68775baab7f9d P 40c77f37a81a464ba7ff42f17bd32c06: No bootstrap required, opened a new log
20:27:41.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:41.492187 4017 ts_tablet_manager.cc:1403] T f9ccfdb32e164e61bde68775baab7f9d P 40c77f37a81a464ba7ff42f17bd32c06: Time spent bootstrapping tablet: real 0.014s user 0.010s sys 0.004s
20:27:41.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:41.493106 4020 tablet_bootstrap.cc:654] T fe829cba01eb4acb9847b261804f81c8 P e149de1a3f47443487db92e7c70143dd: Neither blocks nor log segments found. Creating new log.
20:27:41.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:41.494421 4020 log.cc:826] T fe829cba01eb4acb9847b261804f81c8 P e149de1a3f47443487db92e7c70143dd: Log is configured to *not* fsync() on all Append() calls
20:27:41.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:41.496464 4018 tablet_bootstrap.cc:492] T f9ccfdb32e164e61bde68775baab7f9d P d00c04330aef4a87a22435384a1a5c12: No bootstrap required, opened a new log
20:27:41.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:41.496789 4020 tablet_bootstrap.cc:492] T fe829cba01eb4acb9847b261804f81c8 P e149de1a3f47443487db92e7c70143dd: No bootstrap required, opened a new log
20:27:41.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:41.496779 4018 ts_tablet_manager.cc:1403] T f9ccfdb32e164e61bde68775baab7f9d P d00c04330aef4a87a22435384a1a5c12: Time spent bootstrapping tablet: real 0.015s user 0.006s sys 0.004s
20:27:41.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:41.497061 4020 ts_tablet_manager.cc:1403] T fe829cba01eb4acb9847b261804f81c8 P e149de1a3f47443487db92e7c70143dd: Time spent bootstrapping tablet: real 0.008s user 0.003s sys 0.003s
20:27:41.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:41.499516 4017 raft_consensus.cc:359] T f9ccfdb32e164e61bde68775baab7f9d P 40c77f37a81a464ba7ff42f17bd32c06 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "40c77f37a81a464ba7ff42f17bd32c06" member_type: VOTER last_known_addr { host: "127.3.46.2" port: 37949 } } peers { permanent_uuid: "d00c04330aef4a87a22435384a1a5c12" member_type: VOTER last_known_addr { host: "127.3.46.1" port: 37067 } } peers { permanent_uuid: "e149de1a3f47443487db92e7c70143dd" member_type: VOTER last_known_addr { host: "127.3.46.3" port: 42083 } }
20:27:41.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:41.499943 4017 raft_consensus.cc:385] T f9ccfdb32e164e61bde68775baab7f9d P 40c77f37a81a464ba7ff42f17bd32c06 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
20:27:41.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:41.500048 4017 raft_consensus.cc:740] T f9ccfdb32e164e61bde68775baab7f9d P 40c77f37a81a464ba7ff42f17bd32c06 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 40c77f37a81a464ba7ff42f17bd32c06, State: Initialized, Role: FOLLOWER
20:27:41.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:41.500749 4017 consensus_queue.cc:260] T f9ccfdb32e164e61bde68775baab7f9d P 40c77f37a81a464ba7ff42f17bd32c06 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "40c77f37a81a464ba7ff42f17bd32c06" member_type: VOTER last_known_addr { host: "127.3.46.2" port: 37949 } } peers { permanent_uuid: "d00c04330aef4a87a22435384a1a5c12" member_type: VOTER last_known_addr { host: "127.3.46.1" port: 37067 } } peers { permanent_uuid: "e149de1a3f47443487db92e7c70143dd" member_type: VOTER last_known_addr { host: "127.3.46.3" port: 42083 } }
20:27:41.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:41.502563 4017 ts_tablet_manager.cc:1434] T f9ccfdb32e164e61bde68775baab7f9d P 40c77f37a81a464ba7ff42f17bd32c06: Time spent starting tablet: real 0.010s user 0.006s sys 0.004s
20:27:41.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:41.502715 4020 raft_consensus.cc:359] T fe829cba01eb4acb9847b261804f81c8 P e149de1a3f47443487db92e7c70143dd [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d00c04330aef4a87a22435384a1a5c12" member_type: VOTER last_known_addr { host: "127.3.46.1" port: 37067 } } peers { permanent_uuid: "40c77f37a81a464ba7ff42f17bd32c06" member_type: VOTER last_known_addr { host: "127.3.46.2" port: 37949 } } peers { permanent_uuid: "e149de1a3f47443487db92e7c70143dd" member_type: VOTER last_known_addr { host: "127.3.46.3" port: 42083 } }
20:27:41.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:41.503105 4020 raft_consensus.cc:385] T fe829cba01eb4acb9847b261804f81c8 P e149de1a3f47443487db92e7c70143dd [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
20:27:41.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:41.503211 4020 raft_consensus.cc:740] T fe829cba01eb4acb9847b261804f81c8 P e149de1a3f47443487db92e7c70143dd [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e149de1a3f47443487db92e7c70143dd, State: Initialized, Role: FOLLOWER
20:27:41.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:41.503393 3849 heartbeater.cc:499] Master 127.3.46.62:46071 was elected leader, sending a full tablet report...
20:27:41.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:41.503845 4020 consensus_queue.cc:260] T fe829cba01eb4acb9847b261804f81c8 P e149de1a3f47443487db92e7c70143dd [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d00c04330aef4a87a22435384a1a5c12" member_type: VOTER last_known_addr { host: "127.3.46.1" port: 37067 } } peers { permanent_uuid: "40c77f37a81a464ba7ff42f17bd32c06" member_type: VOTER last_known_addr { host: "127.3.46.2" port: 37949 } } peers { permanent_uuid: "e149de1a3f47443487db92e7c70143dd" member_type: VOTER last_known_addr { host: "127.3.46.3" port: 42083 } }
20:27:41.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:41.506395 4018 raft_consensus.cc:359] T f9ccfdb32e164e61bde68775baab7f9d P d00c04330aef4a87a22435384a1a5c12 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "40c77f37a81a464ba7ff42f17bd32c06" member_type: VOTER last_known_addr { host: "127.3.46.2" port: 37949 } } peers { permanent_uuid: "d00c04330aef4a87a22435384a1a5c12" member_type: VOTER last_known_addr { host: "127.3.46.1" port: 37067 } } peers { permanent_uuid: "e149de1a3f47443487db92e7c70143dd" member_type: VOTER last_known_addr { host: "127.3.46.3" port: 42083 } }
20:27:41.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:41.506803 4018 raft_consensus.cc:385] T f9ccfdb32e164e61bde68775baab7f9d P d00c04330aef4a87a22435384a1a5c12 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
20:27:41.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:41.506922 4018 raft_consensus.cc:740] T f9ccfdb32e164e61bde68775baab7f9d P d00c04330aef4a87a22435384a1a5c12 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: d00c04330aef4a87a22435384a1a5c12, State: Initialized, Role: FOLLOWER
20:27:41.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:41.507773 4018 consensus_queue.cc:260] T f9ccfdb32e164e61bde68775baab7f9d P d00c04330aef4a87a22435384a1a5c12 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "40c77f37a81a464ba7ff42f17bd32c06" member_type: VOTER last_known_addr { host: "127.3.46.2" port: 37949 } } peers { permanent_uuid: "d00c04330aef4a87a22435384a1a5c12" member_type: VOTER last_known_addr { host: "127.3.46.1" port: 37067 } } peers { permanent_uuid: "e149de1a3f47443487db92e7c70143dd" member_type: VOTER last_known_addr { host: "127.3.46.3" port: 42083 } }
20:27:41.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:41.510514 4017 tablet_bootstrap.cc:492] T fe829cba01eb4acb9847b261804f81c8 P 40c77f37a81a464ba7ff42f17bd32c06: Bootstrap starting.
20:27:41.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:41.513523 4018 ts_tablet_manager.cc:1434] T f9ccfdb32e164e61bde68775baab7f9d P d00c04330aef4a87a22435384a1a5c12: Time spent starting tablet: real 0.017s user 0.008s sys 0.005s
20:27:41.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:41.513778 4017 tablet_bootstrap.cc:654] T fe829cba01eb4acb9847b261804f81c8 P 40c77f37a81a464ba7ff42f17bd32c06: Neither blocks nor log segments found. Creating new log.
20:27:41.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:41.514887 4018 tablet_bootstrap.cc:492] T fe829cba01eb4acb9847b261804f81c8 P d00c04330aef4a87a22435384a1a5c12: Bootstrap starting.
20:27:41.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:41.515491 4020 ts_tablet_manager.cc:1434] T fe829cba01eb4acb9847b261804f81c8 P e149de1a3f47443487db92e7c70143dd: Time spent starting tablet: real 0.018s user 0.011s sys 0.008s
20:27:41.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:41.518500 3991 heartbeater.cc:499] Master 127.3.46.62:46071 was elected leader, sending a full tablet report...
20:27:41.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:41.520447 4020 tablet_bootstrap.cc:492] T f9ccfdb32e164e61bde68775baab7f9d P e149de1a3f47443487db92e7c70143dd: Bootstrap starting.
20:27:41.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:41.522604 4020 tablet_bootstrap.cc:654] T f9ccfdb32e164e61bde68775baab7f9d P e149de1a3f47443487db92e7c70143dd: Neither blocks nor log segments found. Creating new log.
20:27:41.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:41.518065 4018 tablet_bootstrap.cc:654] T fe829cba01eb4acb9847b261804f81c8 P d00c04330aef4a87a22435384a1a5c12: Neither blocks nor log segments found. Creating new log.
20:27:41.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:41.527045 4017 tablet_bootstrap.cc:492] T fe829cba01eb4acb9847b261804f81c8 P 40c77f37a81a464ba7ff42f17bd32c06: No bootstrap required, opened a new log
20:27:41.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:41.527400 4017 ts_tablet_manager.cc:1403] T fe829cba01eb4acb9847b261804f81c8 P 40c77f37a81a464ba7ff42f17bd32c06: Time spent bootstrapping tablet: real 0.017s user 0.010s sys 0.004s
20:27:41.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:41.528390 4017 raft_consensus.cc:359] T fe829cba01eb4acb9847b261804f81c8 P 40c77f37a81a464ba7ff42f17bd32c06 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d00c04330aef4a87a22435384a1a5c12" member_type: VOTER last_known_addr { host: "127.3.46.1" port: 37067 } } peers { permanent_uuid: "40c77f37a81a464ba7ff42f17bd32c06" member_type: VOTER last_known_addr { host: "127.3.46.2" port: 37949 } } peers { permanent_uuid: "e149de1a3f47443487db92e7c70143dd" member_type: VOTER last_known_addr { host: "127.3.46.3" port: 42083 } }
20:27:41.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:41.528703 4017 raft_consensus.cc:385] T fe829cba01eb4acb9847b261804f81c8 P 40c77f37a81a464ba7ff42f17bd32c06 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
20:27:41.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:41.528903 4017 raft_consensus.cc:740] T fe829cba01eb4acb9847b261804f81c8 P 40c77f37a81a464ba7ff42f17bd32c06 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 40c77f37a81a464ba7ff42f17bd32c06, State: Initialized, Role: FOLLOWER
20:27:41.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:41.529690 4017 consensus_queue.cc:260] T fe829cba01eb4acb9847b261804f81c8 P 40c77f37a81a464ba7ff42f17bd32c06 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d00c04330aef4a87a22435384a1a5c12" member_type: VOTER last_known_addr { host: "127.3.46.1" port: 37067 } } peers { permanent_uuid: "40c77f37a81a464ba7ff42f17bd32c06" member_type: VOTER last_known_addr { host: "127.3.46.2" port: 37949 } } peers { permanent_uuid: "e149de1a3f47443487db92e7c70143dd" member_type: VOTER last_known_addr { host: "127.3.46.3" port: 42083 } }
20:27:41.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:41.530330 4017 ts_tablet_manager.cc:1434] T fe829cba01eb4acb9847b261804f81c8 P 40c77f37a81a464ba7ff42f17bd32c06: Time spent starting tablet: real 0.003s user 0.002s sys 0.000s
20:27:41.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:41.530730 4020 tablet_bootstrap.cc:492] T f9ccfdb32e164e61bde68775baab7f9d P e149de1a3f47443487db92e7c70143dd: No bootstrap required, opened a new log
20:27:41.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:41.530900 4020 ts_tablet_manager.cc:1403] T f9ccfdb32e164e61bde68775baab7f9d P e149de1a3f47443487db92e7c70143dd: Time spent bootstrapping tablet: real 0.011s user 0.005s sys 0.001s
20:27:41.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:41.532131 4018 tablet_bootstrap.cc:492] T fe829cba01eb4acb9847b261804f81c8 P d00c04330aef4a87a22435384a1a5c12: No bootstrap required, opened a new log
20:27:41.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:41.532315 4018 ts_tablet_manager.cc:1403] T fe829cba01eb4acb9847b261804f81c8 P d00c04330aef4a87a22435384a1a5c12: Time spent bootstrapping tablet: real 0.018s user 0.002s sys 0.005s
20:27:41.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:41.532559 4020 raft_consensus.cc:359] T f9ccfdb32e164e61bde68775baab7f9d P e149de1a3f47443487db92e7c70143dd [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "40c77f37a81a464ba7ff42f17bd32c06" member_type: VOTER last_known_addr { host: "127.3.46.2" port: 37949 } } peers { permanent_uuid: "d00c04330aef4a87a22435384a1a5c12" member_type: VOTER last_known_addr { host: "127.3.46.1" port: 37067 } } peers { permanent_uuid: "e149de1a3f47443487db92e7c70143dd" member_type: VOTER last_known_addr { host: "127.3.46.3" port: 42083 } }
20:27:41.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:41.532837 4020 raft_consensus.cc:385] T f9ccfdb32e164e61bde68775baab7f9d P e149de1a3f47443487db92e7c70143dd [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
20:27:41.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:41.533021 4020 raft_consensus.cc:740] T f9ccfdb32e164e61bde68775baab7f9d P e149de1a3f47443487db92e7c70143dd [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e149de1a3f47443487db92e7c70143dd, State: Initialized, Role: FOLLOWER
20:27:41.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:41.533165 4018 raft_consensus.cc:359] T fe829cba01eb4acb9847b261804f81c8 P d00c04330aef4a87a22435384a1a5c12 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d00c04330aef4a87a22435384a1a5c12" member_type: VOTER last_known_addr { host: "127.3.46.1" port: 37067 } } peers { permanent_uuid: "40c77f37a81a464ba7ff42f17bd32c06" member_type: VOTER last_known_addr { host: "127.3.46.2" port: 37949 } } peers { permanent_uuid: "e149de1a3f47443487db92e7c70143dd" member_type: VOTER last_known_addr { host: "127.3.46.3" port: 42083 } }
20:27:41.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:41.533372 4018 raft_consensus.cc:385] T fe829cba01eb4acb9847b261804f81c8 P d00c04330aef4a87a22435384a1a5c12 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
20:27:41.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:41.533473 4018 raft_consensus.cc:740] T fe829cba01eb4acb9847b261804f81c8 P d00c04330aef4a87a22435384a1a5c12 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: d00c04330aef4a87a22435384a1a5c12, State: Initialized, Role: FOLLOWER
20:27:41.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:41.533519 4020 consensus_queue.cc:260] T f9ccfdb32e164e61bde68775baab7f9d P e149de1a3f47443487db92e7c70143dd [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "40c77f37a81a464ba7ff42f17bd32c06" member_type: VOTER last_known_addr { host: "127.3.46.2" port: 37949 } } peers { permanent_uuid: "d00c04330aef4a87a22435384a1a5c12" member_type: VOTER last_known_addr { host: "127.3.46.1" port: 37067 } } peers { permanent_uuid: "e149de1a3f47443487db92e7c70143dd" member_type: VOTER last_known_addr { host: "127.3.46.3" port: 42083 } }
20:27:41.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:41.533767 4018 consensus_queue.cc:260] T fe829cba01eb4acb9847b261804f81c8 P d00c04330aef4a87a22435384a1a5c12 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d00c04330aef4a87a22435384a1a5c12" member_type: VOTER last_known_addr { host: "127.3.46.1" port: 37067 } } peers { permanent_uuid: "40c77f37a81a464ba7ff42f17bd32c06" member_type: VOTER last_known_addr { host: "127.3.46.2" port: 37949 } } peers { permanent_uuid: "e149de1a3f47443487db92e7c70143dd" member_type: VOTER last_known_addr { host: "127.3.46.3" port: 42083 } }
20:27:41.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:41.534274 4018 ts_tablet_manager.cc:1434] T fe829cba01eb4acb9847b261804f81c8 P d00c04330aef4a87a22435384a1a5c12: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
20:27:41.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:41.534682 4020 ts_tablet_manager.cc:1434] T f9ccfdb32e164e61bde68775baab7f9d P e149de1a3f47443487db92e7c70143dd: Time spent starting tablet: real 0.004s user 0.003s sys 0.000s
20:27:41.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:27:41.536906 3851 tablet.cc:2404] T f9ccfdb32e164e61bde68775baab7f9d P 40c77f37a81a464ba7ff42f17bd32c06: Can't schedule compaction. Clean time has not been advanced past its initial value.
20:27:41.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:27:41.569036 3993 tablet.cc:2404] T f9ccfdb32e164e61bde68775baab7f9d P e149de1a3f47443487db92e7c70143dd: Can't schedule compaction. Clean time has not been advanced past its initial value.
20:27:41.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:41.584771 4025 raft_consensus.cc:493] T f9ccfdb32e164e61bde68775baab7f9d P d00c04330aef4a87a22435384a1a5c12 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
20:27:41.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:41.585176 4025 raft_consensus.cc:515] T f9ccfdb32e164e61bde68775baab7f9d P d00c04330aef4a87a22435384a1a5c12 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "40c77f37a81a464ba7ff42f17bd32c06" member_type: VOTER last_known_addr { host: "127.3.46.2" port: 37949 } } peers { permanent_uuid: "d00c04330aef4a87a22435384a1a5c12" member_type: VOTER last_known_addr { host: "127.3.46.1" port: 37067 } } peers { permanent_uuid: "e149de1a3f47443487db92e7c70143dd" member_type: VOTER last_known_addr { host: "127.3.46.3" port: 42083 } }
20:27:41.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:41.587205 4025 leader_election.cc:290] T f9ccfdb32e164e61bde68775baab7f9d P d00c04330aef4a87a22435384a1a5c12 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 40c77f37a81a464ba7ff42f17bd32c06 (127.3.46.2:37949), e149de1a3f47443487db92e7c70143dd (127.3.46.3:42083)
20:27:41.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:41.595256 3803 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "f9ccfdb32e164e61bde68775baab7f9d" candidate_uuid: "d00c04330aef4a87a22435384a1a5c12" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "40c77f37a81a464ba7ff42f17bd32c06" is_pre_election: true
20:27:41.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:41.595810 3803 raft_consensus.cc:2468] T f9ccfdb32e164e61bde68775baab7f9d P 40c77f37a81a464ba7ff42f17bd32c06 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate d00c04330aef4a87a22435384a1a5c12 in term 0.
20:27:41.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:41.596447 3945 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "f9ccfdb32e164e61bde68775baab7f9d" candidate_uuid: "d00c04330aef4a87a22435384a1a5c12" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e149de1a3f47443487db92e7c70143dd" is_pre_election: true
20:27:41.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:41.596516 3584 leader_election.cc:304] T f9ccfdb32e164e61bde68775baab7f9d P d00c04330aef4a87a22435384a1a5c12 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 40c77f37a81a464ba7ff42f17bd32c06, d00c04330aef4a87a22435384a1a5c12; no voters:
20:27:41.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:41.596881 3945 raft_consensus.cc:2468] T f9ccfdb32e164e61bde68775baab7f9d P e149de1a3f47443487db92e7c70143dd [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate d00c04330aef4a87a22435384a1a5c12 in term 0.
20:27:41.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:41.596940 4025 raft_consensus.cc:2804] T f9ccfdb32e164e61bde68775baab7f9d P d00c04330aef4a87a22435384a1a5c12 [term 0 FOLLOWER]: Leader pre-election won for term 1
20:27:41.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:41.597088 4025 raft_consensus.cc:493] T f9ccfdb32e164e61bde68775baab7f9d P d00c04330aef4a87a22435384a1a5c12 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
20:27:41.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:41.597193 4025 raft_consensus.cc:3060] T f9ccfdb32e164e61bde68775baab7f9d P d00c04330aef4a87a22435384a1a5c12 [term 0 FOLLOWER]: Advancing to term 1
20:27:41.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:41.599238 4025 raft_consensus.cc:515] T f9ccfdb32e164e61bde68775baab7f9d P d00c04330aef4a87a22435384a1a5c12 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "40c77f37a81a464ba7ff42f17bd32c06" member_type: VOTER last_known_addr { host: "127.3.46.2" port: 37949 } } peers { permanent_uuid: "d00c04330aef4a87a22435384a1a5c12" member_type: VOTER last_known_addr { host: "127.3.46.1" port: 37067 } } peers { permanent_uuid: "e149de1a3f47443487db92e7c70143dd" member_type: VOTER last_known_addr { host: "127.3.46.3" port: 42083 } }
20:27:41.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:41.600060 4025 leader_election.cc:290] T f9ccfdb32e164e61bde68775baab7f9d P d00c04330aef4a87a22435384a1a5c12 [CANDIDATE]: Term 1 election: Requested vote from peers 40c77f37a81a464ba7ff42f17bd32c06 (127.3.46.2:37949), e149de1a3f47443487db92e7c70143dd (127.3.46.3:42083)
20:27:41.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:41.600409 3803 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "f9ccfdb32e164e61bde68775baab7f9d" candidate_uuid: "d00c04330aef4a87a22435384a1a5c12" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "40c77f37a81a464ba7ff42f17bd32c06"
20:27:41.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:41.600631 3945 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "f9ccfdb32e164e61bde68775baab7f9d" candidate_uuid: "d00c04330aef4a87a22435384a1a5c12" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e149de1a3f47443487db92e7c70143dd"
20:27:41.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:41.600636 3803 raft_consensus.cc:3060] T f9ccfdb32e164e61bde68775baab7f9d P 40c77f37a81a464ba7ff42f17bd32c06 [term 0 FOLLOWER]: Advancing to term 1
20:27:41.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:41.600842 3945 raft_consensus.cc:3060] T f9ccfdb32e164e61bde68775baab7f9d P e149de1a3f47443487db92e7c70143dd [term 0 FOLLOWER]: Advancing to term 1
20:27:41.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:41.603009 3803 raft_consensus.cc:2468] T f9ccfdb32e164e61bde68775baab7f9d P 40c77f37a81a464ba7ff42f17bd32c06 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate d00c04330aef4a87a22435384a1a5c12 in term 1.
20:27:41.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:41.603080 3945 raft_consensus.cc:2468] T f9ccfdb32e164e61bde68775baab7f9d P e149de1a3f47443487db92e7c70143dd [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate d00c04330aef4a87a22435384a1a5c12 in term 1.
20:27:41.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:41.603449 3584 leader_election.cc:304] T f9ccfdb32e164e61bde68775baab7f9d P d00c04330aef4a87a22435384a1a5c12 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 40c77f37a81a464ba7ff42f17bd32c06, d00c04330aef4a87a22435384a1a5c12; no voters:
20:27:41.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:41.603740 4025 raft_consensus.cc:2804] T f9ccfdb32e164e61bde68775baab7f9d P d00c04330aef4a87a22435384a1a5c12 [term 1 FOLLOWER]: Leader election won for term 1
20:27:41.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:41.604265 4025 raft_consensus.cc:697] T f9ccfdb32e164e61bde68775baab7f9d P d00c04330aef4a87a22435384a1a5c12 [term 1 LEADER]: Becoming Leader. State: Replica: d00c04330aef4a87a22435384a1a5c12, State: Running, Role: LEADER
20:27:41.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:41.604820 4025 consensus_queue.cc:237] T f9ccfdb32e164e61bde68775baab7f9d P d00c04330aef4a87a22435384a1a5c12 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "40c77f37a81a464ba7ff42f17bd32c06" member_type: VOTER last_known_addr { host: "127.3.46.2" port: 37949 } } peers { permanent_uuid: "d00c04330aef4a87a22435384a1a5c12" member_type: VOTER last_known_addr { host: "127.3.46.1" port: 37067 } } peers { permanent_uuid: "e149de1a3f47443487db92e7c70143dd" member_type: VOTER last_known_addr { host: "127.3.46.3" port: 42083 } }
20:27:41.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:41.610242 3296 catalog_manager.cc:5671] T f9ccfdb32e164e61bde68775baab7f9d P d00c04330aef4a87a22435384a1a5c12 reported cstate change: term changed from 0 to 1, leader changed from <none> to d00c04330aef4a87a22435384a1a5c12 (127.3.46.1). New cstate: current_term: 1 leader_uuid: "d00c04330aef4a87a22435384a1a5c12" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "40c77f37a81a464ba7ff42f17bd32c06" member_type: VOTER last_known_addr { host: "127.3.46.2" port: 37949 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "d00c04330aef4a87a22435384a1a5c12" member_type: VOTER last_known_addr { host: "127.3.46.1" port: 37067 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "e149de1a3f47443487db92e7c70143dd" member_type: VOTER last_known_addr { host: "127.3.46.3" port: 42083 } health_report { overall_health: UNKNOWN } } }
20:27:41.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:27:41.723289 3709 tablet.cc:2404] T f9ccfdb32e164e61bde68775baab7f9d P d00c04330aef4a87a22435384a1a5c12: Can't schedule compaction. Clean time has not been advanced past its initial value.
20:27:41.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:41.773380 4025 raft_consensus.cc:493] T fe829cba01eb4acb9847b261804f81c8 P d00c04330aef4a87a22435384a1a5c12 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
20:27:41.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:41.773607 4025 raft_consensus.cc:515] T fe829cba01eb4acb9847b261804f81c8 P d00c04330aef4a87a22435384a1a5c12 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d00c04330aef4a87a22435384a1a5c12" member_type: VOTER last_known_addr { host: "127.3.46.1" port: 37067 } } peers { permanent_uuid: "40c77f37a81a464ba7ff42f17bd32c06" member_type: VOTER last_known_addr { host: "127.3.46.2" port: 37949 } } peers { permanent_uuid: "e149de1a3f47443487db92e7c70143dd" member_type: VOTER last_known_addr { host: "127.3.46.3" port: 42083 } }
20:27:41.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:41.774273 4025 leader_election.cc:290] T fe829cba01eb4acb9847b261804f81c8 P d00c04330aef4a87a22435384a1a5c12 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 40c77f37a81a464ba7ff42f17bd32c06 (127.3.46.2:37949), e149de1a3f47443487db92e7c70143dd (127.3.46.3:42083)
20:27:41.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:41.774855 3803 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "fe829cba01eb4acb9847b261804f81c8" candidate_uuid: "d00c04330aef4a87a22435384a1a5c12" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "40c77f37a81a464ba7ff42f17bd32c06" is_pre_election: true
20:27:41.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:41.774951 3945 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "fe829cba01eb4acb9847b261804f81c8" candidate_uuid: "d00c04330aef4a87a22435384a1a5c12" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e149de1a3f47443487db92e7c70143dd" is_pre_election: true
20:27:41.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:41.775092 3803 raft_consensus.cc:2468] T fe829cba01eb4acb9847b261804f81c8 P 40c77f37a81a464ba7ff42f17bd32c06 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate d00c04330aef4a87a22435384a1a5c12 in term 0.
20:27:41.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:41.775229 3945 raft_consensus.cc:2468] T fe829cba01eb4acb9847b261804f81c8 P e149de1a3f47443487db92e7c70143dd [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate d00c04330aef4a87a22435384a1a5c12 in term 0.
20:27:41.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:41.775612 3584 leader_election.cc:304] T fe829cba01eb4acb9847b261804f81c8 P d00c04330aef4a87a22435384a1a5c12 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 40c77f37a81a464ba7ff42f17bd32c06, d00c04330aef4a87a22435384a1a5c12; no voters:
20:27:41.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:41.775952 4025 raft_consensus.cc:2804] T fe829cba01eb4acb9847b261804f81c8 P d00c04330aef4a87a22435384a1a5c12 [term 0 FOLLOWER]: Leader pre-election won for term 1
20:27:41.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:41.776059 4025 raft_consensus.cc:493] T fe829cba01eb4acb9847b261804f81c8 P d00c04330aef4a87a22435384a1a5c12 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
20:27:41.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:41.776168 4025 raft_consensus.cc:3060] T fe829cba01eb4acb9847b261804f81c8 P d00c04330aef4a87a22435384a1a5c12 [term 0 FOLLOWER]: Advancing to term 1
20:27:41.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:41.778092 4025 raft_consensus.cc:515] T fe829cba01eb4acb9847b261804f81c8 P d00c04330aef4a87a22435384a1a5c12 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d00c04330aef4a87a22435384a1a5c12" member_type: VOTER last_known_addr { host: "127.3.46.1" port: 37067 } } peers { permanent_uuid: "40c77f37a81a464ba7ff42f17bd32c06" member_type: VOTER last_known_addr { host: "127.3.46.2" port: 37949 } } peers { permanent_uuid: "e149de1a3f47443487db92e7c70143dd" member_type: VOTER last_known_addr { host: "127.3.46.3" port: 42083 } }
20:27:41.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:41.778733 4025 leader_election.cc:290] T fe829cba01eb4acb9847b261804f81c8 P d00c04330aef4a87a22435384a1a5c12 [CANDIDATE]: Term 1 election: Requested vote from peers 40c77f37a81a464ba7ff42f17bd32c06 (127.3.46.2:37949), e149de1a3f47443487db92e7c70143dd (127.3.46.3:42083)
20:27:41.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:41.779312 3803 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "fe829cba01eb4acb9847b261804f81c8" candidate_uuid: "d00c04330aef4a87a22435384a1a5c12" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "40c77f37a81a464ba7ff42f17bd32c06"
20:27:41.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:41.779440 3945 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "fe829cba01eb4acb9847b261804f81c8" candidate_uuid: "d00c04330aef4a87a22435384a1a5c12" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e149de1a3f47443487db92e7c70143dd"
20:27:41.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:41.779546 3803 raft_consensus.cc:3060] T fe829cba01eb4acb9847b261804f81c8 P 40c77f37a81a464ba7ff42f17bd32c06 [term 0 FOLLOWER]: Advancing to term 1
20:27:41.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:41.779620 3945 raft_consensus.cc:3060] T fe829cba01eb4acb9847b261804f81c8 P e149de1a3f47443487db92e7c70143dd [term 0 FOLLOWER]: Advancing to term 1
20:27:41.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:41.781554 3803 raft_consensus.cc:2468] T fe829cba01eb4acb9847b261804f81c8 P 40c77f37a81a464ba7ff42f17bd32c06 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate d00c04330aef4a87a22435384a1a5c12 in term 1.
20:27:41.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:41.781585 3945 raft_consensus.cc:2468] T fe829cba01eb4acb9847b261804f81c8 P e149de1a3f47443487db92e7c70143dd [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate d00c04330aef4a87a22435384a1a5c12 in term 1.
20:27:41.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:41.782050 3584 leader_election.cc:304] T fe829cba01eb4acb9847b261804f81c8 P d00c04330aef4a87a22435384a1a5c12 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: d00c04330aef4a87a22435384a1a5c12, e149de1a3f47443487db92e7c70143dd; no voters:
20:27:41.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:41.782348 4025 raft_consensus.cc:2804] T fe829cba01eb4acb9847b261804f81c8 P d00c04330aef4a87a22435384a1a5c12 [term 1 FOLLOWER]: Leader election won for term 1
20:27:41.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:41.782502 4025 raft_consensus.cc:697] T fe829cba01eb4acb9847b261804f81c8 P d00c04330aef4a87a22435384a1a5c12 [term 1 LEADER]: Becoming Leader. State: Replica: d00c04330aef4a87a22435384a1a5c12, State: Running, Role: LEADER
20:27:41.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:41.782778 4025 consensus_queue.cc:237] T fe829cba01eb4acb9847b261804f81c8 P d00c04330aef4a87a22435384a1a5c12 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d00c04330aef4a87a22435384a1a5c12" member_type: VOTER last_known_addr { host: "127.3.46.1" port: 37067 } } peers { permanent_uuid: "40c77f37a81a464ba7ff42f17bd32c06" member_type: VOTER last_known_addr { host: "127.3.46.2" port: 37949 } } peers { permanent_uuid: "e149de1a3f47443487db92e7c70143dd" member_type: VOTER last_known_addr { host: "127.3.46.3" port: 42083 } }
20:27:41.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:41.785574 3296 catalog_manager.cc:5671] T fe829cba01eb4acb9847b261804f81c8 P d00c04330aef4a87a22435384a1a5c12 reported cstate change: term changed from 0 to 1, leader changed from <none> to d00c04330aef4a87a22435384a1a5c12 (127.3.46.1). New cstate: current_term: 1 leader_uuid: "d00c04330aef4a87a22435384a1a5c12" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d00c04330aef4a87a22435384a1a5c12" member_type: VOTER last_known_addr { host: "127.3.46.1" port: 37067 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "40c77f37a81a464ba7ff42f17bd32c06" member_type: VOTER last_known_addr { host: "127.3.46.2" port: 37949 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "e149de1a3f47443487db92e7c70143dd" member_type: VOTER last_known_addr { host: "127.3.46.3" port: 42083 } health_report { overall_health: UNKNOWN } } }
20:27:42.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:42.039939 4025 consensus_queue.cc:1048] T f9ccfdb32e164e61bde68775baab7f9d P d00c04330aef4a87a22435384a1a5c12 [LEADER]: Connected to new peer: Peer: permanent_uuid: "e149de1a3f47443487db92e7c70143dd" member_type: VOTER last_known_addr { host: "127.3.46.3" port: 42083 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
20:27:42.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:42.049033 4025 consensus_queue.cc:1048] T f9ccfdb32e164e61bde68775baab7f9d P d00c04330aef4a87a22435384a1a5c12 [LEADER]: Connected to new peer: Peer: permanent_uuid: "40c77f37a81a464ba7ff42f17bd32c06" member_type: VOTER last_known_addr { host: "127.3.46.2" port: 37949 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
20:27:42.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:42.092020 3296 catalog_manager.cc:2257] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47558:
20:27:42.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "kudu_system.kudu_transactions"
20:27:42.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
20:27:42.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
20:27:42.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
20:27:42.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT64
20:27:42.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
20:27:42.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
20:27:42.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
20:27:42.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
20:27:42.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
20:27:42.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
20:27:42.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
20:27:42.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
20:27:42.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "entry_type"
20:27:42.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT8
20:27:42.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
20:27:42.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
20:27:42.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
20:27:42.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
20:27:42.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
20:27:42.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
20:27:42.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
20:27:42.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
20:27:42.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "identifier"
20:27:42.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
20:27:42.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
20:27:42.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
20:27:42.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
20:27:42.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
20:27:42.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
20:27:42.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
20:27:42.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
20:27:42.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
20:27:42.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "metadata"
20:27:42.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
20:27:42.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
20:27:42.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
20:27:42.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
20:27:42.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
20:27:42.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
20:27:42.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
20:27:42.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
20:27:42.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
20:27:42.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_replicas: 3
20:27:42.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) split_rows_range_bounds {
20:27:42.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) rows: "<redacted>""\006\001\000\000\000\000\000\000\000\000\007\001@B\017\000\000\000\000\000"
20:27:42.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) indirect_data: "<redacted>"""
20:27:42.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
20:27:42.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
20:27:42.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) range_schema {
20:27:42.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
20:27:42.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
20:27:42.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
20:27:42.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
20:27:42.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
20:27:42.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) table_type: TXN_STATUS_TABLE
20:27:42.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:27:42.093307 3296 catalog_manager.cc:7033] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table kudu_system.kudu_transactions in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
20:27:42.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:42.105757 3779 tablet_service.cc:1511] Processing CreateTablet for tablet ce76969b09994fd7b9068b7f105e8fc4 (TXN_STATUS_TABLE table=kudu_system.kudu_transactions [id=aca98b500f164adba6b0ae55bd806e83]), partition=RANGE (txn_id) PARTITION 0 <= VALUES < 1000000
20:27:42.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:42.106091 3924 tablet_service.cc:1511] Processing CreateTablet for tablet ce76969b09994fd7b9068b7f105e8fc4 (TXN_STATUS_TABLE table=kudu_system.kudu_transactions [id=aca98b500f164adba6b0ae55bd806e83]), partition=RANGE (txn_id) PARTITION 0 <= VALUES < 1000000
20:27:42.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:42.106875 3779 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet ce76969b09994fd7b9068b7f105e8fc4. 1 dirs total, 0 dirs full, 0 dirs failed
20:27:42.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:42.106885 3924 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet ce76969b09994fd7b9068b7f105e8fc4. 1 dirs total, 0 dirs full, 0 dirs failed
20:27:42.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:42.113963 3641 tablet_service.cc:1511] Processing CreateTablet for tablet ce76969b09994fd7b9068b7f105e8fc4 (TXN_STATUS_TABLE table=kudu_system.kudu_transactions [id=aca98b500f164adba6b0ae55bd806e83]), partition=RANGE (txn_id) PARTITION 0 <= VALUES < 1000000
20:27:42.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:42.114989 3641 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet ce76969b09994fd7b9068b7f105e8fc4. 1 dirs total, 0 dirs full, 0 dirs failed
20:27:42.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:42.115911 4053 tablet_bootstrap.cc:492] T ce76969b09994fd7b9068b7f105e8fc4 P e149de1a3f47443487db92e7c70143dd: Bootstrap starting.
20:27:42.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:42.117671 4053 tablet_bootstrap.cc:654] T ce76969b09994fd7b9068b7f105e8fc4 P e149de1a3f47443487db92e7c70143dd: Neither blocks nor log segments found. Creating new log.
20:27:42.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:42.119571 4053 tablet_bootstrap.cc:492] T ce76969b09994fd7b9068b7f105e8fc4 P e149de1a3f47443487db92e7c70143dd: No bootstrap required, opened a new log
20:27:42.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:42.119689 4053 ts_tablet_manager.cc:1403] T ce76969b09994fd7b9068b7f105e8fc4 P e149de1a3f47443487db92e7c70143dd: Time spent bootstrapping tablet: real 0.004s user 0.003s sys 0.000s
20:27:42.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:42.120437 4053 raft_consensus.cc:359] T ce76969b09994fd7b9068b7f105e8fc4 P e149de1a3f47443487db92e7c70143dd [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d00c04330aef4a87a22435384a1a5c12" member_type: VOTER last_known_addr { host: "127.3.46.1" port: 37067 } } peers { permanent_uuid: "40c77f37a81a464ba7ff42f17bd32c06" member_type: VOTER last_known_addr { host: "127.3.46.2" port: 37949 } } peers { permanent_uuid: "e149de1a3f47443487db92e7c70143dd" member_type: VOTER last_known_addr { host: "127.3.46.3" port: 42083 } }
20:27:42.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:42.120632 4053 raft_consensus.cc:385] T ce76969b09994fd7b9068b7f105e8fc4 P e149de1a3f47443487db92e7c70143dd [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
20:27:42.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:42.120716 4053 raft_consensus.cc:740] T ce76969b09994fd7b9068b7f105e8fc4 P e149de1a3f47443487db92e7c70143dd [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e149de1a3f47443487db92e7c70143dd, State: Initialized, Role: FOLLOWER
20:27:42.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:42.120769 4055 tablet_bootstrap.cc:492] T ce76969b09994fd7b9068b7f105e8fc4 P d00c04330aef4a87a22435384a1a5c12: Bootstrap starting.
20:27:42.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:42.121050 4053 consensus_queue.cc:260] T ce76969b09994fd7b9068b7f105e8fc4 P e149de1a3f47443487db92e7c70143dd [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d00c04330aef4a87a22435384a1a5c12" member_type: VOTER last_known_addr { host: "127.3.46.1" port: 37067 } } peers { permanent_uuid: "40c77f37a81a464ba7ff42f17bd32c06" member_type: VOTER last_known_addr { host: "127.3.46.2" port: 37949 } } peers { permanent_uuid: "e149de1a3f47443487db92e7c70143dd" member_type: VOTER last_known_addr { host: "127.3.46.3" port: 42083 } }
20:27:42.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:42.121587 4053 ts_tablet_manager.cc:1434] T ce76969b09994fd7b9068b7f105e8fc4 P e149de1a3f47443487db92e7c70143dd: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
20:27:42.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:42.121526 4034 tablet_replica.cc:442] T ce76969b09994fd7b9068b7f105e8fc4 P e149de1a3f47443487db92e7c70143dd: TxnStatusTablet state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d00c04330aef4a87a22435384a1a5c12" member_type: VOTER last_known_addr { host: "127.3.46.1" port: 37067 } } peers { permanent_uuid: "40c77f37a81a464ba7ff42f17bd32c06" member_type: VOTER last_known_addr { host: "127.3.46.2" port: 37949 } } peers { permanent_uuid: "e149de1a3f47443487db92e7c70143dd" member_type: VOTER last_known_addr { host: "127.3.46.3" port: 42083 } } }
20:27:42.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:42.121824 4034 tablet_replica.cc:445] T ce76969b09994fd7b9068b7f105e8fc4 P e149de1a3f47443487db92e7c70143dd: This TxnStatusTablet replica's current role is: FOLLOWER
20:27:42.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:42.124361 4055 tablet_bootstrap.cc:654] T ce76969b09994fd7b9068b7f105e8fc4 P d00c04330aef4a87a22435384a1a5c12: Neither blocks nor log segments found. Creating new log.
20:27:42.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:42.126194 4056 tablet_bootstrap.cc:492] T ce76969b09994fd7b9068b7f105e8fc4 P 40c77f37a81a464ba7ff42f17bd32c06: Bootstrap starting.
20:27:42.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:42.126331 4055 tablet_bootstrap.cc:492] T ce76969b09994fd7b9068b7f105e8fc4 P d00c04330aef4a87a22435384a1a5c12: No bootstrap required, opened a new log
20:27:42.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:42.126475 4055 ts_tablet_manager.cc:1403] T ce76969b09994fd7b9068b7f105e8fc4 P d00c04330aef4a87a22435384a1a5c12: Time spent bootstrapping tablet: real 0.006s user 0.003s sys 0.000s
20:27:42.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:42.127187 4055 raft_consensus.cc:359] T ce76969b09994fd7b9068b7f105e8fc4 P d00c04330aef4a87a22435384a1a5c12 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d00c04330aef4a87a22435384a1a5c12" member_type: VOTER last_known_addr { host: "127.3.46.1" port: 37067 } } peers { permanent_uuid: "40c77f37a81a464ba7ff42f17bd32c06" member_type: VOTER last_known_addr { host: "127.3.46.2" port: 37949 } } peers { permanent_uuid: "e149de1a3f47443487db92e7c70143dd" member_type: VOTER last_known_addr { host: "127.3.46.3" port: 42083 } }
20:27:42.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:42.127339 4055 raft_consensus.cc:385] T ce76969b09994fd7b9068b7f105e8fc4 P d00c04330aef4a87a22435384a1a5c12 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
20:27:42.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:42.127422 4055 raft_consensus.cc:740] T ce76969b09994fd7b9068b7f105e8fc4 P d00c04330aef4a87a22435384a1a5c12 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: d00c04330aef4a87a22435384a1a5c12, State: Initialized, Role: FOLLOWER
20:27:42.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:42.127686 4055 consensus_queue.cc:260] T ce76969b09994fd7b9068b7f105e8fc4 P d00c04330aef4a87a22435384a1a5c12 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d00c04330aef4a87a22435384a1a5c12" member_type: VOTER last_known_addr { host: "127.3.46.1" port: 37067 } } peers { permanent_uuid: "40c77f37a81a464ba7ff42f17bd32c06" member_type: VOTER last_known_addr { host: "127.3.46.2" port: 37949 } } peers { permanent_uuid: "e149de1a3f47443487db92e7c70143dd" member_type: VOTER last_known_addr { host: "127.3.46.3" port: 42083 } }
20:27:42.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:42.127854 4056 tablet_bootstrap.cc:654] T ce76969b09994fd7b9068b7f105e8fc4 P 40c77f37a81a464ba7ff42f17bd32c06: Neither blocks nor log segments found. Creating new log.
20:27:42.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:42.128252 4055 ts_tablet_manager.cc:1434] T ce76969b09994fd7b9068b7f105e8fc4 P d00c04330aef4a87a22435384a1a5c12: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
20:27:42.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:42.128187 4025 tablet_replica.cc:442] T ce76969b09994fd7b9068b7f105e8fc4 P d00c04330aef4a87a22435384a1a5c12: TxnStatusTablet state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d00c04330aef4a87a22435384a1a5c12" member_type: VOTER last_known_addr { host: "127.3.46.1" port: 37067 } } peers { permanent_uuid: "40c77f37a81a464ba7ff42f17bd32c06" member_type: VOTER last_known_addr { host: "127.3.46.2" port: 37949 } } peers { permanent_uuid: "e149de1a3f47443487db92e7c70143dd" member_type: VOTER last_known_addr { host: "127.3.46.3" port: 42083 } } }
20:27:42.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:42.128420 4025 tablet_replica.cc:445] T ce76969b09994fd7b9068b7f105e8fc4 P d00c04330aef4a87a22435384a1a5c12: This TxnStatusTablet replica's current role is: FOLLOWER
20:27:42.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:42.130818 4056 tablet_bootstrap.cc:492] T ce76969b09994fd7b9068b7f105e8fc4 P 40c77f37a81a464ba7ff42f17bd32c06: No bootstrap required, opened a new log
20:27:42.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:42.130992 4056 ts_tablet_manager.cc:1403] T ce76969b09994fd7b9068b7f105e8fc4 P 40c77f37a81a464ba7ff42f17bd32c06: Time spent bootstrapping tablet: real 0.005s user 0.003s sys 0.000s
20:27:42.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:42.131809 4056 raft_consensus.cc:359] T ce76969b09994fd7b9068b7f105e8fc4 P 40c77f37a81a464ba7ff42f17bd32c06 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d00c04330aef4a87a22435384a1a5c12" member_type: VOTER last_known_addr { host: "127.3.46.1" port: 37067 } } peers { permanent_uuid: "40c77f37a81a464ba7ff42f17bd32c06" member_type: VOTER last_known_addr { host: "127.3.46.2" port: 37949 } } peers { permanent_uuid: "e149de1a3f47443487db92e7c70143dd" member_type: VOTER last_known_addr { host: "127.3.46.3" port: 42083 } }
20:27:42.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:42.132007 4056 raft_consensus.cc:385] T ce76969b09994fd7b9068b7f105e8fc4 P 40c77f37a81a464ba7ff42f17bd32c06 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
20:27:42.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:42.132093 4056 raft_consensus.cc:740] T ce76969b09994fd7b9068b7f105e8fc4 P 40c77f37a81a464ba7ff42f17bd32c06 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 40c77f37a81a464ba7ff42f17bd32c06, State: Initialized, Role: FOLLOWER
20:27:42.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:42.132639 4056 consensus_queue.cc:260] T ce76969b09994fd7b9068b7f105e8fc4 P 40c77f37a81a464ba7ff42f17bd32c06 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d00c04330aef4a87a22435384a1a5c12" member_type: VOTER last_known_addr { host: "127.3.46.1" port: 37067 } } peers { permanent_uuid: "40c77f37a81a464ba7ff42f17bd32c06" member_type: VOTER last_known_addr { host: "127.3.46.2" port: 37949 } } peers { permanent_uuid: "e149de1a3f47443487db92e7c70143dd" member_type: VOTER last_known_addr { host: "127.3.46.3" port: 42083 } }
20:27:42.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:42.133177 4056 ts_tablet_manager.cc:1434] T ce76969b09994fd7b9068b7f105e8fc4 P 40c77f37a81a464ba7ff42f17bd32c06: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
20:27:42.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:42.133147 4039 tablet_replica.cc:442] T ce76969b09994fd7b9068b7f105e8fc4 P 40c77f37a81a464ba7ff42f17bd32c06: TxnStatusTablet state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d00c04330aef4a87a22435384a1a5c12" member_type: VOTER last_known_addr { host: "127.3.46.1" port: 37067 } } peers { permanent_uuid: "40c77f37a81a464ba7ff42f17bd32c06" member_type: VOTER last_known_addr { host: "127.3.46.2" port: 37949 } } peers { permanent_uuid: "e149de1a3f47443487db92e7c70143dd" member_type: VOTER last_known_addr { host: "127.3.46.3" port: 42083 } } }
20:27:42.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:42.133455 4039 tablet_replica.cc:445] T ce76969b09994fd7b9068b7f105e8fc4 P 40c77f37a81a464ba7ff42f17bd32c06: This TxnStatusTablet replica's current role is: FOLLOWER
20:27:42.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:42.134388 4034 raft_consensus.cc:493] T ce76969b09994fd7b9068b7f105e8fc4 P e149de1a3f47443487db92e7c70143dd [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
20:27:42.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:42.134562 4034 raft_consensus.cc:515] T ce76969b09994fd7b9068b7f105e8fc4 P e149de1a3f47443487db92e7c70143dd [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d00c04330aef4a87a22435384a1a5c12" member_type: VOTER last_known_addr { host: "127.3.46.1" port: 37067 } } peers { permanent_uuid: "40c77f37a81a464ba7ff42f17bd32c06" member_type: VOTER last_known_addr { host: "127.3.46.2" port: 37949 } } peers { permanent_uuid: "e149de1a3f47443487db92e7c70143dd" member_type: VOTER last_known_addr { host: "127.3.46.3" port: 42083 } }
20:27:42.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:42.136600 4034 leader_election.cc:290] T ce76969b09994fd7b9068b7f105e8fc4 P e149de1a3f47443487db92e7c70143dd [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers d00c04330aef4a87a22435384a1a5c12 (127.3.46.1:37067), 40c77f37a81a464ba7ff42f17bd32c06 (127.3.46.2:37949)
20:27:42.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:42.149358 3661 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "ce76969b09994fd7b9068b7f105e8fc4" candidate_uuid: "e149de1a3f47443487db92e7c70143dd" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d00c04330aef4a87a22435384a1a5c12" is_pre_election: true
20:27:42.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:42.149772 3661 raft_consensus.cc:2468] T ce76969b09994fd7b9068b7f105e8fc4 P d00c04330aef4a87a22435384a1a5c12 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate e149de1a3f47443487db92e7c70143dd in term 0.
20:27:42.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:42.150063 3803 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "ce76969b09994fd7b9068b7f105e8fc4" candidate_uuid: "e149de1a3f47443487db92e7c70143dd" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "40c77f37a81a464ba7ff42f17bd32c06" is_pre_election: true
20:27:42.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:42.150306 3803 raft_consensus.cc:2468] T ce76969b09994fd7b9068b7f105e8fc4 P 40c77f37a81a464ba7ff42f17bd32c06 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate e149de1a3f47443487db92e7c70143dd in term 0.
20:27:42.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:42.150389 3868 leader_election.cc:304] T ce76969b09994fd7b9068b7f105e8fc4 P e149de1a3f47443487db92e7c70143dd [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: d00c04330aef4a87a22435384a1a5c12, e149de1a3f47443487db92e7c70143dd; no voters:
20:27:42.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:42.150795 4034 raft_consensus.cc:2804] T ce76969b09994fd7b9068b7f105e8fc4 P e149de1a3f47443487db92e7c70143dd [term 0 FOLLOWER]: Leader pre-election won for term 1
20:27:42.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:42.150906 4034 raft_consensus.cc:493] T ce76969b09994fd7b9068b7f105e8fc4 P e149de1a3f47443487db92e7c70143dd [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
20:27:42.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:42.150979 4034 raft_consensus.cc:3060] T ce76969b09994fd7b9068b7f105e8fc4 P e149de1a3f47443487db92e7c70143dd [term 0 FOLLOWER]: Advancing to term 1
20:27:42.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:42.152863 4034 raft_consensus.cc:515] T ce76969b09994fd7b9068b7f105e8fc4 P e149de1a3f47443487db92e7c70143dd [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d00c04330aef4a87a22435384a1a5c12" member_type: VOTER last_known_addr { host: "127.3.46.1" port: 37067 } } peers { permanent_uuid: "40c77f37a81a464ba7ff42f17bd32c06" member_type: VOTER last_known_addr { host: "127.3.46.2" port: 37949 } } peers { permanent_uuid: "e149de1a3f47443487db92e7c70143dd" member_type: VOTER last_known_addr { host: "127.3.46.3" port: 42083 } }
20:27:42.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:42.153441 4034 leader_election.cc:290] T ce76969b09994fd7b9068b7f105e8fc4 P e149de1a3f47443487db92e7c70143dd [CANDIDATE]: Term 1 election: Requested vote from peers d00c04330aef4a87a22435384a1a5c12 (127.3.46.1:37067), 40c77f37a81a464ba7ff42f17bd32c06 (127.3.46.2:37949)
20:27:42.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:42.154078 3661 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "ce76969b09994fd7b9068b7f105e8fc4" candidate_uuid: "e149de1a3f47443487db92e7c70143dd" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d00c04330aef4a87a22435384a1a5c12"
20:27:42.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:42.154071 3803 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "ce76969b09994fd7b9068b7f105e8fc4" candidate_uuid: "e149de1a3f47443487db92e7c70143dd" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "40c77f37a81a464ba7ff42f17bd32c06"
20:27:42.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:42.154281 3661 raft_consensus.cc:3060] T ce76969b09994fd7b9068b7f105e8fc4 P d00c04330aef4a87a22435384a1a5c12 [term 0 FOLLOWER]: Advancing to term 1
20:27:42.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:42.154311 3803 raft_consensus.cc:3060] T ce76969b09994fd7b9068b7f105e8fc4 P 40c77f37a81a464ba7ff42f17bd32c06 [term 0 FOLLOWER]: Advancing to term 1
20:27:42.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:42.156209 3803 raft_consensus.cc:2468] T ce76969b09994fd7b9068b7f105e8fc4 P 40c77f37a81a464ba7ff42f17bd32c06 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate e149de1a3f47443487db92e7c70143dd in term 1.
20:27:42.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:42.156383 3661 raft_consensus.cc:2468] T ce76969b09994fd7b9068b7f105e8fc4 P d00c04330aef4a87a22435384a1a5c12 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate e149de1a3f47443487db92e7c70143dd in term 1.
20:27:42.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:42.156653 3870 leader_election.cc:304] T ce76969b09994fd7b9068b7f105e8fc4 P e149de1a3f47443487db92e7c70143dd [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 40c77f37a81a464ba7ff42f17bd32c06, e149de1a3f47443487db92e7c70143dd; no voters:
20:27:42.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:42.157061 4034 raft_consensus.cc:2804] T ce76969b09994fd7b9068b7f105e8fc4 P e149de1a3f47443487db92e7c70143dd [term 1 FOLLOWER]: Leader election won for term 1
20:27:42.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:42.157615 4034 raft_consensus.cc:697] T ce76969b09994fd7b9068b7f105e8fc4 P e149de1a3f47443487db92e7c70143dd [term 1 LEADER]: Becoming Leader. State: Replica: e149de1a3f47443487db92e7c70143dd, State: Running, Role: LEADER
20:27:42.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:42.158207 4034 consensus_queue.cc:237] T ce76969b09994fd7b9068b7f105e8fc4 P e149de1a3f47443487db92e7c70143dd [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d00c04330aef4a87a22435384a1a5c12" member_type: VOTER last_known_addr { host: "127.3.46.1" port: 37067 } } peers { permanent_uuid: "40c77f37a81a464ba7ff42f17bd32c06" member_type: VOTER last_known_addr { host: "127.3.46.2" port: 37949 } } peers { permanent_uuid: "e149de1a3f47443487db92e7c70143dd" member_type: VOTER last_known_addr { host: "127.3.46.3" port: 42083 } }
20:27:42.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:42.160936 4063 tablet_replica.cc:442] T ce76969b09994fd7b9068b7f105e8fc4 P e149de1a3f47443487db92e7c70143dd: TxnStatusTablet state changed. Reason: New leader e149de1a3f47443487db92e7c70143dd. Latest consensus state: current_term: 1 leader_uuid: "e149de1a3f47443487db92e7c70143dd" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d00c04330aef4a87a22435384a1a5c12" member_type: VOTER last_known_addr { host: "127.3.46.1" port: 37067 } } peers { permanent_uuid: "40c77f37a81a464ba7ff42f17bd32c06" member_type: VOTER last_known_addr { host: "127.3.46.2" port: 37949 } } peers { permanent_uuid: "e149de1a3f47443487db92e7c70143dd" member_type: VOTER last_known_addr { host: "127.3.46.3" port: 42083 } } }
20:27:42.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:42.161237 4063 tablet_replica.cc:445] T ce76969b09994fd7b9068b7f105e8fc4 P e149de1a3f47443487db92e7c70143dd: This TxnStatusTablet replica's current role is: LEADER
20:27:42.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:42.162446 3296 catalog_manager.cc:5671] T ce76969b09994fd7b9068b7f105e8fc4 P e149de1a3f47443487db92e7c70143dd reported cstate change: term changed from 0 to 1, leader changed from <none> to e149de1a3f47443487db92e7c70143dd (127.3.46.3). New cstate: current_term: 1 leader_uuid: "e149de1a3f47443487db92e7c70143dd" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d00c04330aef4a87a22435384a1a5c12" member_type: VOTER last_known_addr { host: "127.3.46.1" port: 37067 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "40c77f37a81a464ba7ff42f17bd32c06" member_type: VOTER last_known_addr { host: "127.3.46.2" port: 37949 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "e149de1a3f47443487db92e7c70143dd" member_type: VOTER last_known_addr { host: "127.3.46.3" port: 42083 } health_report { overall_health: HEALTHY } } }
20:27:42.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:42.163058 4065 txn_status_manager.cc:874] Waiting until node catch up with all replicated operations in previous term...
20:27:42.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:42.163267 4065 txn_status_manager.cc:930] Loading transaction status metadata into memory...
20:27:42.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:42.198802 4025 consensus_queue.cc:1048] T fe829cba01eb4acb9847b261804f81c8 P d00c04330aef4a87a22435384a1a5c12 [LEADER]: Connected to new peer: Peer: permanent_uuid: "40c77f37a81a464ba7ff42f17bd32c06" member_type: VOTER last_known_addr { host: "127.3.46.2" port: 37949 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
20:27:42.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:42.203701 3661 raft_consensus.cc:1275] T ce76969b09994fd7b9068b7f105e8fc4 P d00c04330aef4a87a22435384a1a5c12 [term 1 FOLLOWER]: Refusing update from remote peer e149de1a3f47443487db92e7c70143dd: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
20:27:42.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:42.203879 3802 raft_consensus.cc:1275] T ce76969b09994fd7b9068b7f105e8fc4 P 40c77f37a81a464ba7ff42f17bd32c06 [term 1 FOLLOWER]: Refusing update from remote peer e149de1a3f47443487db92e7c70143dd: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
20:27:42.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:42.204614 4034 consensus_queue.cc:1048] T ce76969b09994fd7b9068b7f105e8fc4 P e149de1a3f47443487db92e7c70143dd [LEADER]: Connected to new peer: Peer: permanent_uuid: "d00c04330aef4a87a22435384a1a5c12" member_type: VOTER last_known_addr { host: "127.3.46.1" port: 37067 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
20:27:42.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:42.205621 4034 consensus_queue.cc:1048] T ce76969b09994fd7b9068b7f105e8fc4 P e149de1a3f47443487db92e7c70143dd [LEADER]: Connected to new peer: Peer: permanent_uuid: "40c77f37a81a464ba7ff42f17bd32c06" member_type: VOTER last_known_addr { host: "127.3.46.2" port: 37949 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
20:27:42.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:42.210100 4037 consensus_queue.cc:1048] T fe829cba01eb4acb9847b261804f81c8 P d00c04330aef4a87a22435384a1a5c12 [LEADER]: Connected to new peer: Peer: permanent_uuid: "e149de1a3f47443487db92e7c70143dd" member_type: VOTER last_known_addr { host: "127.3.46.3" port: 42083 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
20:27:42.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:42.228866 4039 tablet_replica.cc:442] T ce76969b09994fd7b9068b7f105e8fc4 P 40c77f37a81a464ba7ff42f17bd32c06: TxnStatusTablet state changed. Reason: New leader e149de1a3f47443487db92e7c70143dd. Latest consensus state: current_term: 1 leader_uuid: "e149de1a3f47443487db92e7c70143dd" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d00c04330aef4a87a22435384a1a5c12" member_type: VOTER last_known_addr { host: "127.3.46.1" port: 37067 } } peers { permanent_uuid: "40c77f37a81a464ba7ff42f17bd32c06" member_type: VOTER last_known_addr { host: "127.3.46.2" port: 37949 } } peers { permanent_uuid: "e149de1a3f47443487db92e7c70143dd" member_type: VOTER last_known_addr { host: "127.3.46.3" port: 42083 } } }
20:27:42.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:42.229153 4039 tablet_replica.cc:445] T ce76969b09994fd7b9068b7f105e8fc4 P 40c77f37a81a464ba7ff42f17bd32c06: This TxnStatusTablet replica's current role is: FOLLOWER
20:27:42.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:42.237807 4038 mvcc.cc:204] Tried to move back new op lower bound from 7281325105974571008 to 7281325105804722176. Current Snapshot: MvccSnapshot[applied={T|T < 7281325105974571008}]
20:27:42.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:42.238631 4041 mvcc.cc:204] Tried to move back new op lower bound from 7281325105974571008 to 7281325105804722176. Current Snapshot: MvccSnapshot[applied={T|T < 7281325105974571008}]
20:27:42.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:42.246919 4025 tablet_replica.cc:442] T ce76969b09994fd7b9068b7f105e8fc4 P d00c04330aef4a87a22435384a1a5c12: TxnStatusTablet state changed. Reason: New leader e149de1a3f47443487db92e7c70143dd. Latest consensus state: current_term: 1 leader_uuid: "e149de1a3f47443487db92e7c70143dd" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d00c04330aef4a87a22435384a1a5c12" member_type: VOTER last_known_addr { host: "127.3.46.1" port: 37067 } } peers { permanent_uuid: "40c77f37a81a464ba7ff42f17bd32c06" member_type: VOTER last_known_addr { host: "127.3.46.2" port: 37949 } } peers { permanent_uuid: "e149de1a3f47443487db92e7c70143dd" member_type: VOTER last_known_addr { host: "127.3.46.3" port: 42083 } } }
20:27:42.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:42.247150 4025 tablet_replica.cc:445] T ce76969b09994fd7b9068b7f105e8fc4 P d00c04330aef4a87a22435384a1a5c12: This TxnStatusTablet replica's current role is: FOLLOWER
20:27:42.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:42.250708 4036 mvcc.cc:204] Tried to move back new op lower bound from 7281325105974571008 to 7281325105804722176. Current Snapshot: MvccSnapshot[applied={T|T < 7281325105974571008}]
20:27:42.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:42.255383 4039 tablet_replica.cc:442] T ce76969b09994fd7b9068b7f105e8fc4 P 40c77f37a81a464ba7ff42f17bd32c06: TxnStatusTablet state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "e149de1a3f47443487db92e7c70143dd" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d00c04330aef4a87a22435384a1a5c12" member_type: VOTER last_known_addr { host: "127.3.46.1" port: 37067 } } peers { permanent_uuid: "40c77f37a81a464ba7ff42f17bd32c06" member_type: VOTER last_known_addr { host: "127.3.46.2" port: 37949 } } peers { permanent_uuid: "e149de1a3f47443487db92e7c70143dd" member_type: VOTER last_known_addr { host: "127.3.46.3" port: 42083 } } }
20:27:42.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:42.255600 4039 tablet_replica.cc:445] T ce76969b09994fd7b9068b7f105e8fc4 P 40c77f37a81a464ba7ff42f17bd32c06: This TxnStatusTablet replica's current role is: FOLLOWER
20:27:42.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:42.260685 4034 tablet_replica.cc:442] T ce76969b09994fd7b9068b7f105e8fc4 P e149de1a3f47443487db92e7c70143dd: TxnStatusTablet state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "e149de1a3f47443487db92e7c70143dd" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d00c04330aef4a87a22435384a1a5c12" member_type: VOTER last_known_addr { host: "127.3.46.1" port: 37067 } } peers { permanent_uuid: "40c77f37a81a464ba7ff42f17bd32c06" member_type: VOTER last_known_addr { host: "127.3.46.2" port: 37949 } } peers { permanent_uuid: "e149de1a3f47443487db92e7c70143dd" member_type: VOTER last_known_addr { host: "127.3.46.3" port: 42083 } } }
20:27:42.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:42.260979 4034 tablet_replica.cc:445] T ce76969b09994fd7b9068b7f105e8fc4 P e149de1a3f47443487db92e7c70143dd: This TxnStatusTablet replica's current role is: LEADER
20:27:42.263 [DEBUG - Time-limited test] (KuduTransaction.java:702) starting keepalive heartbeating with period 500 ms (txn ID 0)
20:27:42.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:42.264220 4034 tablet_replica.cc:442] T ce76969b09994fd7b9068b7f105e8fc4 P e149de1a3f47443487db92e7c70143dd: TxnStatusTablet state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "e149de1a3f47443487db92e7c70143dd" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d00c04330aef4a87a22435384a1a5c12" member_type: VOTER last_known_addr { host: "127.3.46.1" port: 37067 } } peers { permanent_uuid: "40c77f37a81a464ba7ff42f17bd32c06" member_type: VOTER last_known_addr { host: "127.3.46.2" port: 37949 } } peers { permanent_uuid: "e149de1a3f47443487db92e7c70143dd" member_type: VOTER last_known_addr { host: "127.3.46.3" port: 42083 } } }
20:27:42.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:42.264490 4034 tablet_replica.cc:445] T ce76969b09994fd7b9068b7f105e8fc4 P e149de1a3f47443487db92e7c70143dd: This TxnStatusTablet replica's current role is: LEADER
20:27:42.269 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:894) Opened table 346e6e6149384e3992393d04f9f16b9c
20:27:42.269 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.3.46.62:46071
20:27:42.269 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
20:27:42.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:42.269696 4025 tablet_replica.cc:442] T ce76969b09994fd7b9068b7f105e8fc4 P d00c04330aef4a87a22435384a1a5c12: TxnStatusTablet state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "e149de1a3f47443487db92e7c70143dd" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d00c04330aef4a87a22435384a1a5c12" member_type: VOTER last_known_addr { host: "127.3.46.1" port: 37067 } } peers { permanent_uuid: "40c77f37a81a464ba7ff42f17bd32c06" member_type: VOTER last_known_addr { host: "127.3.46.2" port: 37949 } } peers { permanent_uuid: "e149de1a3f47443487db92e7c70143dd" member_type: VOTER last_known_addr { host: "127.3.46.3" port: 42083 } } }
20:27:42.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:42.269960 4025 tablet_replica.cc:445] T ce76969b09994fd7b9068b7f105e8fc4 P d00c04330aef4a87a22435384a1a5c12: This TxnStatusTablet replica's current role is: FOLLOWER
20:27:42.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:42.269953 3256 external_mini_cluster.cc:1699] Attempting to check leaks for /tmp/dist-test-taska94UBC/build/asan/bin/kudu pid 3267
20:27:42.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:42.271376 3338 generic_service.cc:196] Checking for leaks (request via RPC)
20:27:42.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:42.430375 3256 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taska94UBC/build/asan/bin/kudu with pid 3267
20:27:42.459 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.3.46.62:46071(127.3.46.62:46071)] handling channelInactive
20:27:42.459 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.62:46071(127.3.46.62:46071)] cleaning up while in state READY due to: connection closed
20:27:42.460 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:27:42.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:27:42.538679 3707 heartbeater.cc:646] Failed to heartbeat to 127.3.46.62:46071 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.3.46.62:46071: connect: Connection refused (error 111)
20:27:42.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:42.730857 4078 raft_consensus.cc:493] T 00000000000000000000000000000000 P b1731f8308c145f686f88efd6a54b851 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader 0db1d539946942f794b0c4ec81f8f4b4)
20:27:42.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:42.730851 4079 raft_consensus.cc:493] T 00000000000000000000000000000000 P d6c270b387b04f17b2a58bfcb0eb4f82 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader 0db1d539946942f794b0c4ec81f8f4b4)
20:27:42.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:42.731106 4079 raft_consensus.cc:515] T 00000000000000000000000000000000 P d6c270b387b04f17b2a58bfcb0eb4f82 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0db1d539946942f794b0c4ec81f8f4b4" member_type: VOTER last_known_addr { host: "127.3.46.62" port: 46071 } } peers { permanent_uuid: "d6c270b387b04f17b2a58bfcb0eb4f82" member_type: VOTER last_known_addr { host: "127.3.46.61" port: 39407 } } peers { permanent_uuid: "b1731f8308c145f686f88efd6a54b851" member_type: VOTER last_known_addr { host: "127.3.46.60" port: 36127 } }
20:27:42.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:42.731091 4078 raft_consensus.cc:515] T 00000000000000000000000000000000 P b1731f8308c145f686f88efd6a54b851 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0db1d539946942f794b0c4ec81f8f4b4" member_type: VOTER last_known_addr { host: "127.3.46.62" port: 46071 } } peers { permanent_uuid: "d6c270b387b04f17b2a58bfcb0eb4f82" member_type: VOTER last_known_addr { host: "127.3.46.61" port: 39407 } } peers { permanent_uuid: "b1731f8308c145f686f88efd6a54b851" member_type: VOTER last_known_addr { host: "127.3.46.60" port: 36127 } }
20:27:42.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:42.731946 4079 leader_election.cc:290] T 00000000000000000000000000000000 P d6c270b387b04f17b2a58bfcb0eb4f82 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 0db1d539946942f794b0c4ec81f8f4b4 (127.3.46.62:46071), b1731f8308c145f686f88efd6a54b851 (127.3.46.60:36127)
20:27:42.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:42.732517 4078 leader_election.cc:290] T 00000000000000000000000000000000 P b1731f8308c145f686f88efd6a54b851 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 0db1d539946942f794b0c4ec81f8f4b4 (127.3.46.62:46071), d6c270b387b04f17b2a58bfcb0eb4f82 (127.3.46.61:39407)
20:27:42.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:42.732566 3469 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "d6c270b387b04f17b2a58bfcb0eb4f82" candidate_term: 2 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "b1731f8308c145f686f88efd6a54b851" is_pre_election: true
20:27:42.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:42.732864 3469 raft_consensus.cc:2468] T 00000000000000000000000000000000 P b1731f8308c145f686f88efd6a54b851 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate d6c270b387b04f17b2a58bfcb0eb4f82 in term 1.
20:27:42.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:42.733690 3389 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "b1731f8308c145f686f88efd6a54b851" candidate_term: 2 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "d6c270b387b04f17b2a58bfcb0eb4f82" is_pre_election: true
20:27:42.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:42.733691 3363 leader_election.cc:304] T 00000000000000000000000000000000 P d6c270b387b04f17b2a58bfcb0eb4f82 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: b1731f8308c145f686f88efd6a54b851, d6c270b387b04f17b2a58bfcb0eb4f82; no voters:
20:27:42.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:42.734025 3389 raft_consensus.cc:2468] T 00000000000000000000000000000000 P d6c270b387b04f17b2a58bfcb0eb4f82 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate b1731f8308c145f686f88efd6a54b851 in term 1.
20:27:42.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:42.734153 4079 raft_consensus.cc:2804] T 00000000000000000000000000000000 P d6c270b387b04f17b2a58bfcb0eb4f82 [term 1 FOLLOWER]: Leader pre-election won for term 2
20:27:42.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:27:42.734133 3445 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.3.46.62:46071: connect: Connection refused (error 111)
20:27:42.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:42.734261 4079 raft_consensus.cc:493] T 00000000000000000000000000000000 P d6c270b387b04f17b2a58bfcb0eb4f82 [term 1 FOLLOWER]: Starting leader election (detected failure of leader 0db1d539946942f794b0c4ec81f8f4b4)
20:27:42.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:42.734357 4079 raft_consensus.cc:3060] T 00000000000000000000000000000000 P d6c270b387b04f17b2a58bfcb0eb4f82 [term 1 FOLLOWER]: Advancing to term 2
20:27:42.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:27:42.734671 3364 leader_election.cc:336] T 00000000000000000000000000000000 P d6c270b387b04f17b2a58bfcb0eb4f82 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 0db1d539946942f794b0c4ec81f8f4b4 (127.3.46.62:46071): Network error: Client connection negotiation failed: client connection to 127.3.46.62:46071: connect: Connection refused (error 111)
20:27:42.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:42.735688 3445 leader_election.cc:304] T 00000000000000000000000000000000 P b1731f8308c145f686f88efd6a54b851 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: b1731f8308c145f686f88efd6a54b851, d6c270b387b04f17b2a58bfcb0eb4f82; no voters:
20:27:42.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:42.736101 4078 raft_consensus.cc:2804] T 00000000000000000000000000000000 P b1731f8308c145f686f88efd6a54b851 [term 1 FOLLOWER]: Leader pre-election won for term 2
20:27:42.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:42.736332 4078 raft_consensus.cc:493] T 00000000000000000000000000000000 P b1731f8308c145f686f88efd6a54b851 [term 1 FOLLOWER]: Starting leader election (detected failure of leader 0db1d539946942f794b0c4ec81f8f4b4)
20:27:42.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:42.736443 4078 raft_consensus.cc:3060] T 00000000000000000000000000000000 P b1731f8308c145f686f88efd6a54b851 [term 1 FOLLOWER]: Advancing to term 2
20:27:42.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:27:42.736650 3445 leader_election.cc:336] T 00000000000000000000000000000000 P b1731f8308c145f686f88efd6a54b851 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 0db1d539946942f794b0c4ec81f8f4b4 (127.3.46.62:46071): Network error: Client connection negotiation failed: client connection to 127.3.46.62:46071: connect: Connection refused (error 111)
20:27:42.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:42.736862 4079 raft_consensus.cc:515] T 00000000000000000000000000000000 P d6c270b387b04f17b2a58bfcb0eb4f82 [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0db1d539946942f794b0c4ec81f8f4b4" member_type: VOTER last_known_addr { host: "127.3.46.62" port: 46071 } } peers { permanent_uuid: "d6c270b387b04f17b2a58bfcb0eb4f82" member_type: VOTER last_known_addr { host: "127.3.46.61" port: 39407 } } peers { permanent_uuid: "b1731f8308c145f686f88efd6a54b851" member_type: VOTER last_known_addr { host: "127.3.46.60" port: 36127 } }
20:27:42.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:42.737567 4079 leader_election.cc:290] T 00000000000000000000000000000000 P d6c270b387b04f17b2a58bfcb0eb4f82 [CANDIDATE]: Term 2 election: Requested vote from peers 0db1d539946942f794b0c4ec81f8f4b4 (127.3.46.62:46071), b1731f8308c145f686f88efd6a54b851 (127.3.46.60:36127)
20:27:42.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:42.738083 3469 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "d6c270b387b04f17b2a58bfcb0eb4f82" candidate_term: 2 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "b1731f8308c145f686f88efd6a54b851"
20:27:42.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:42.739032 4078 raft_consensus.cc:515] T 00000000000000000000000000000000 P b1731f8308c145f686f88efd6a54b851 [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0db1d539946942f794b0c4ec81f8f4b4" member_type: VOTER last_known_addr { host: "127.3.46.62" port: 46071 } } peers { permanent_uuid: "d6c270b387b04f17b2a58bfcb0eb4f82" member_type: VOTER last_known_addr { host: "127.3.46.61" port: 39407 } } peers { permanent_uuid: "b1731f8308c145f686f88efd6a54b851" member_type: VOTER last_known_addr { host: "127.3.46.60" port: 36127 } }
20:27:42.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:27:42.739117 3364 leader_election.cc:336] T 00000000000000000000000000000000 P d6c270b387b04f17b2a58bfcb0eb4f82 [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer 0db1d539946942f794b0c4ec81f8f4b4 (127.3.46.62:46071): Network error: Client connection negotiation failed: client connection to 127.3.46.62:46071: connect: Connection refused (error 111)
20:27:42.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:42.739449 3469 raft_consensus.cc:2393] T 00000000000000000000000000000000 P b1731f8308c145f686f88efd6a54b851 [term 2 FOLLOWER]: Leader election vote request: Denying vote to candidate d6c270b387b04f17b2a58bfcb0eb4f82 in current term 2: Already voted for candidate b1731f8308c145f686f88efd6a54b851 in this term.
20:27:42.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:42.739699 4078 leader_election.cc:290] T 00000000000000000000000000000000 P b1731f8308c145f686f88efd6a54b851 [CANDIDATE]: Term 2 election: Requested vote from peers 0db1d539946942f794b0c4ec81f8f4b4 (127.3.46.62:46071), d6c270b387b04f17b2a58bfcb0eb4f82 (127.3.46.61:39407)
20:27:42.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:42.740411 3363 leader_election.cc:304] T 00000000000000000000000000000000 P d6c270b387b04f17b2a58bfcb0eb4f82 [CANDIDATE]: Term 2 election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: d6c270b387b04f17b2a58bfcb0eb4f82; no voters: 0db1d539946942f794b0c4ec81f8f4b4, b1731f8308c145f686f88efd6a54b851
20:27:42.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:42.740453 3389 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "b1731f8308c145f686f88efd6a54b851" candidate_term: 2 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "d6c270b387b04f17b2a58bfcb0eb4f82"
20:27:42.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:42.740764 3389 raft_consensus.cc:2393] T 00000000000000000000000000000000 P d6c270b387b04f17b2a58bfcb0eb4f82 [term 2 FOLLOWER]: Leader election vote request: Denying vote to candidate b1731f8308c145f686f88efd6a54b851 in current term 2: Already voted for candidate d6c270b387b04f17b2a58bfcb0eb4f82 in this term.
20:27:42.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:42.740949 4079 raft_consensus.cc:2749] T 00000000000000000000000000000000 P d6c270b387b04f17b2a58bfcb0eb4f82 [term 2 FOLLOWER]: Leader election lost for term 2. Reason: could not achieve majority
20:27:42.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:27:42.741055 3445 leader_election.cc:336] T 00000000000000000000000000000000 P b1731f8308c145f686f88efd6a54b851 [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer 0db1d539946942f794b0c4ec81f8f4b4 (127.3.46.62:46071): Network error: Client connection negotiation failed: client connection to 127.3.46.62:46071: connect: Connection refused (error 111)
20:27:42.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:42.741334 3445 leader_election.cc:304] T 00000000000000000000000000000000 P b1731f8308c145f686f88efd6a54b851 [CANDIDATE]: Term 2 election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: b1731f8308c145f686f88efd6a54b851; no voters: 0db1d539946942f794b0c4ec81f8f4b4, d6c270b387b04f17b2a58bfcb0eb4f82
20:27:42.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:42.741600 4078 raft_consensus.cc:2749] T 00000000000000000000000000000000 P b1731f8308c145f686f88efd6a54b851 [term 2 FOLLOWER]: Leader election lost for term 2. Reason: could not achieve majority
20:27:42.771 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.62:46071(127.3.46.62:46071)] connecting to peer
20:27:42.772 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.62:46071(127.3.46.62:46071): Connection refused: /127.3.46.62:46071
20:27:42.772 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.62:46071(127.3.46.62:46071)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.62:46071(127.3.46.62:46071): Connection refused: /127.3.46.62:46071
20:27:42.772 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location master-127.3.46.62:46071(127.3.46.62:46071) for tablet Kudu Master: Failed to connect to peer master-127.3.46.62:46071(127.3.46.62:46071): Connection refused: /127.3.46.62:46071
20:27:42.791 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.62:46071(127.3.46.62:46071)] connecting to peer
20:27:42.793 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.62:46071(127.3.46.62:46071): Connection refused: /127.3.46.62:46071
20:27:42.793 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.62:46071(127.3.46.62:46071)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.62:46071(127.3.46.62:46071): Connection refused: /127.3.46.62:46071
20:27:42.793 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.62:46071: Failed to connect to peer master-127.3.46.62:46071(127.3.46.62:46071): Connection refused: /127.3.46.62:46071
20:27:42.794 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:42.810 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.62:46071(127.3.46.62:46071)] connecting to peer
20:27:42.812 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.62:46071(127.3.46.62:46071): Connection refused: /127.3.46.62:46071
20:27:42.812 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.62:46071(127.3.46.62:46071)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.62:46071(127.3.46.62:46071): Connection refused: /127.3.46.62:46071
20:27:42.812 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.62:46071: Failed to connect to peer master-127.3.46.62:46071(127.3.46.62:46071): Connection refused: /127.3.46.62:46071
20:27:42.813 [WARN - kudu-nio-2] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:42.830 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.62:46071(127.3.46.62:46071)] connecting to peer
20:27:42.832 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.62:46071(127.3.46.62:46071): Connection refused: /127.3.46.62:46071
20:27:42.832 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.62:46071(127.3.46.62:46071)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.62:46071(127.3.46.62:46071): Connection refused: /127.3.46.62:46071
20:27:42.832 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.62:46071: Failed to connect to peer master-127.3.46.62:46071(127.3.46.62:46071): Connection refused: /127.3.46.62:46071
20:27:42.833 [WARN - kudu-nio-6] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:42.851 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.62:46071(127.3.46.62:46071)] connecting to peer
20:27:42.852 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.62:46071(127.3.46.62:46071): Connection refused: /127.3.46.62:46071
20:27:42.852 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.62:46071(127.3.46.62:46071)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.62:46071(127.3.46.62:46071): Connection refused: /127.3.46.62:46071
20:27:42.853 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.62:46071: Failed to connect to peer master-127.3.46.62:46071(127.3.46.62:46071): Connection refused: /127.3.46.62:46071
20:27:42.853 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:42.890 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.62:46071(127.3.46.62:46071)] connecting to peer
20:27:42.891 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.62:46071(127.3.46.62:46071): Connection refused: /127.3.46.62:46071
20:27:42.891 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.62:46071(127.3.46.62:46071)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.62:46071(127.3.46.62:46071): Connection refused: /127.3.46.62:46071
20:27:42.891 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.62:46071: Failed to connect to peer master-127.3.46.62:46071(127.3.46.62:46071): Connection refused: /127.3.46.62:46071
20:27:42.892 [WARN - kudu-nio-2] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:42.931 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.62:46071(127.3.46.62:46071)] connecting to peer
20:27:42.932 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.62:46071(127.3.46.62:46071): Connection refused: /127.3.46.62:46071
20:27:42.932 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.62:46071(127.3.46.62:46071)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.62:46071(127.3.46.62:46071): Connection refused: /127.3.46.62:46071
20:27:42.932 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.62:46071: Failed to connect to peer master-127.3.46.62:46071(127.3.46.62:46071): Connection refused: /127.3.46.62:46071
20:27:42.933 [WARN - kudu-nio-2] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:43.031 [DEBUG - pool-40-thread-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=7, TimeoutTracker(timeout=250, elapsed=259), Traces: [0ms] sending RPC to server master-127.3.46.62:46071, [2ms] received response from server master-127.3.46.62:46071: Network error: Failed to connect to peer master-127.3.46.62:46071(127.3.46.62:46071): Connection refused: /127.3.46.62:46071, [2ms] delaying RPC due to: Network error: Failed to connect to peer master-127.3.46.62:46071(127.3.46.62:46071): Connection refused: /127.3.46.62:46071, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [22ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [22ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [23ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: Network error: Failed to connect to peer master-127.3.46.62:46071(127.3.46.62:46071): Connection refused: /127.3.46.62:46071, [23ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: OK, [24ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: OK, [24ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.62:46071(127.3.46.62:46071): Connection refused: /127.3.46.62:46071, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: Network error: Failed to connect to peer master-127.3.46.62:46071(127.3.46.62:46071): Connection refused: /127.3.46.62:46071, [43ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: OK, [43ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.62:46071(127.3.46.62:46071): Connection refused: /127.3.46.62:46071, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: OK, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: OK, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: Network error: Failed to connect to peer master-127.3.46.62:46071(127.3.46.62:46071): Connection refused: /127.3.46.62:46071, [63ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.62:46071(127.3.46.62:46071): Connection refused: /127.3.46.62:46071, [80ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [82ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: Network error: Failed to connect to peer master-127.3.46.62:46071(127.3.46.62:46071): Connection refused: /127.3.46.62:46071, [83ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: OK, [83ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: OK, [84ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.62:46071(127.3.46.62:46071): Connection refused: /127.3.46.62:46071, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: Network error: Failed to connect to peer master-127.3.46.62:46071(127.3.46.62:46071): Connection refused: /127.3.46.62:46071, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: OK, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.62:46071(127.3.46.62:46071): Connection refused: /127.3.46.62:46071, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: Network error: Failed to connect to peer master-127.3.46.62:46071(127.3.46.62:46071): Connection refused: /127.3.46.62:46071, [163ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: OK, [163ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.62:46071(127.3.46.62:46071): Connection refused: /127.3.46.62:46071, deferred=Deferred@1436767915(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$233/0x00007f8b98336cd0@6d5c4460 -> (continuation of Deferred@1833288354 after retry RPC after error@890774726) -> (continuation of Deferred@2059479744 after retry RPC after error@1966153404) -> (continuation of Deferred@908231274 after retry RPC after error@1114160764) -> (continuation of Deferred@831265755 after retry RPC after error@1278556072) -> (continuation of Deferred@1856425366 after retry RPC after error@1773798752) -> (continuation of Deferred@943294799 after retry RPC after error@1574203799), errback=org.apache.kudu.client.KuduTransaction$$Lambda$234/0x00007f8b98336ef8@546cdc6 -> (continuation of Deferred@1833288354 after retry RPC after error@890774726) -> (continuation of Deferred@2059479744 after retry RPC after error@1966153404) -> (continuation of Deferred@908231274 after retry RPC after error@1114160764) -> (continuation of Deferred@831265755 after retry RPC after error@1278556072) -> (continuation of Deferred@1856425366 after retry RPC after error@1773798752) -> (continuation of Deferred@943294799 after retry RPC after error@1574203799)))
20:27:43.032 [WARN - pool-40-thread-1] (HashedWheelTimer.java:716) An exception was thrown by TimerTask.
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713)
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701)
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:787)
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501)
at java.base/java.lang.Thread.run(Thread.java:840)
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713)
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701)
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:787)
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501)
at java.base/java.lang.Thread.run(Thread.java:840)
20:27:43.051 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.62:46071(127.3.46.62:46071)] connecting to peer
20:27:43.052 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.62:46071(127.3.46.62:46071): Connection refused: /127.3.46.62:46071
20:27:43.052 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.62:46071(127.3.46.62:46071)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.62:46071(127.3.46.62:46071): Connection refused: /127.3.46.62:46071
20:27:43.052 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.62:46071: Failed to connect to peer master-127.3.46.62:46071(127.3.46.62:46071): Connection refused: /127.3.46.62:46071
20:27:43.053 [WARN - kudu-nio-2] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:43.071 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.62:46071(127.3.46.62:46071)] connecting to peer
20:27:43.072 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.62:46071(127.3.46.62:46071): Connection refused: /127.3.46.62:46071
20:27:43.072 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.62:46071(127.3.46.62:46071)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.62:46071(127.3.46.62:46071): Connection refused: /127.3.46.62:46071
20:27:43.072 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.62:46071: Failed to connect to peer master-127.3.46.62:46071(127.3.46.62:46071): Connection refused: /127.3.46.62:46071
20:27:43.073 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:43.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:43.077859 4079 raft_consensus.cc:493] T 00000000000000000000000000000000 P d6c270b387b04f17b2a58bfcb0eb4f82 [term 2 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
20:27:43.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:43.078140 4079 raft_consensus.cc:515] T 00000000000000000000000000000000 P d6c270b387b04f17b2a58bfcb0eb4f82 [term 2 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0db1d539946942f794b0c4ec81f8f4b4" member_type: VOTER last_known_addr { host: "127.3.46.62" port: 46071 } } peers { permanent_uuid: "d6c270b387b04f17b2a58bfcb0eb4f82" member_type: VOTER last_known_addr { host: "127.3.46.61" port: 39407 } } peers { permanent_uuid: "b1731f8308c145f686f88efd6a54b851" member_type: VOTER last_known_addr { host: "127.3.46.60" port: 36127 } }
20:27:43.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:43.078848 4079 leader_election.cc:290] T 00000000000000000000000000000000 P d6c270b387b04f17b2a58bfcb0eb4f82 [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers 0db1d539946942f794b0c4ec81f8f4b4 (127.3.46.62:46071), b1731f8308c145f686f88efd6a54b851 (127.3.46.60:36127)
20:27:43.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:43.079520 3469 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "d6c270b387b04f17b2a58bfcb0eb4f82" candidate_term: 3 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "b1731f8308c145f686f88efd6a54b851" is_pre_election: true
20:27:43.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:43.079818 3469 raft_consensus.cc:2468] T 00000000000000000000000000000000 P b1731f8308c145f686f88efd6a54b851 [term 2 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate d6c270b387b04f17b2a58bfcb0eb4f82 in term 2.
20:27:43.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:43.080255 3363 leader_election.cc:304] T 00000000000000000000000000000000 P d6c270b387b04f17b2a58bfcb0eb4f82 [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: b1731f8308c145f686f88efd6a54b851, d6c270b387b04f17b2a58bfcb0eb4f82; no voters:
20:27:43.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:27:43.080451 3364 leader_election.cc:336] T 00000000000000000000000000000000 P d6c270b387b04f17b2a58bfcb0eb4f82 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer 0db1d539946942f794b0c4ec81f8f4b4 (127.3.46.62:46071): Network error: Client connection negotiation failed: client connection to 127.3.46.62:46071: connect: Connection refused (error 111)
20:27:43.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:43.080549 4079 raft_consensus.cc:2804] T 00000000000000000000000000000000 P d6c270b387b04f17b2a58bfcb0eb4f82 [term 2 FOLLOWER]: Leader pre-election won for term 3
20:27:43.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:43.080649 4079 raft_consensus.cc:493] T 00000000000000000000000000000000 P d6c270b387b04f17b2a58bfcb0eb4f82 [term 2 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
20:27:43.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:43.080760 4079 raft_consensus.cc:3060] T 00000000000000000000000000000000 P d6c270b387b04f17b2a58bfcb0eb4f82 [term 2 FOLLOWER]: Advancing to term 3
20:27:43.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:43.082683 4079 raft_consensus.cc:515] T 00000000000000000000000000000000 P d6c270b387b04f17b2a58bfcb0eb4f82 [term 3 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0db1d539946942f794b0c4ec81f8f4b4" member_type: VOTER last_known_addr { host: "127.3.46.62" port: 46071 } } peers { permanent_uuid: "d6c270b387b04f17b2a58bfcb0eb4f82" member_type: VOTER last_known_addr { host: "127.3.46.61" port: 39407 } } peers { permanent_uuid: "b1731f8308c145f686f88efd6a54b851" member_type: VOTER last_known_addr { host: "127.3.46.60" port: 36127 } }
20:27:43.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:43.083263 4079 leader_election.cc:290] T 00000000000000000000000000000000 P d6c270b387b04f17b2a58bfcb0eb4f82 [CANDIDATE]: Term 3 election: Requested vote from peers 0db1d539946942f794b0c4ec81f8f4b4 (127.3.46.62:46071), b1731f8308c145f686f88efd6a54b851 (127.3.46.60:36127)
20:27:43.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:43.083761 3469 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "d6c270b387b04f17b2a58bfcb0eb4f82" candidate_term: 3 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "b1731f8308c145f686f88efd6a54b851"
20:27:43.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:43.083992 3469 raft_consensus.cc:3060] T 00000000000000000000000000000000 P b1731f8308c145f686f88efd6a54b851 [term 2 FOLLOWER]: Advancing to term 3
20:27:43.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:27:43.084554 3364 leader_election.cc:336] T 00000000000000000000000000000000 P d6c270b387b04f17b2a58bfcb0eb4f82 [CANDIDATE]: Term 3 election: RPC error from VoteRequest() call to peer 0db1d539946942f794b0c4ec81f8f4b4 (127.3.46.62:46071): Network error: Client connection negotiation failed: client connection to 127.3.46.62:46071: connect: Connection refused (error 111)
20:27:43.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:43.085878 3469 raft_consensus.cc:2468] T 00000000000000000000000000000000 P b1731f8308c145f686f88efd6a54b851 [term 3 FOLLOWER]: Leader election vote request: Granting yes vote for candidate d6c270b387b04f17b2a58bfcb0eb4f82 in term 3.
20:27:43.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:43.086287 3363 leader_election.cc:304] T 00000000000000000000000000000000 P d6c270b387b04f17b2a58bfcb0eb4f82 [CANDIDATE]: Term 3 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: b1731f8308c145f686f88efd6a54b851, d6c270b387b04f17b2a58bfcb0eb4f82; no voters: 0db1d539946942f794b0c4ec81f8f4b4
20:27:43.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:43.086596 4079 raft_consensus.cc:2804] T 00000000000000000000000000000000 P d6c270b387b04f17b2a58bfcb0eb4f82 [term 3 FOLLOWER]: Leader election won for term 3
20:27:43.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:43.087059 4079 raft_consensus.cc:697] T 00000000000000000000000000000000 P d6c270b387b04f17b2a58bfcb0eb4f82 [term 3 LEADER]: Becoming Leader. State: Replica: d6c270b387b04f17b2a58bfcb0eb4f82, State: Running, Role: LEADER
20:27:43.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:43.087489 4079 consensus_queue.cc:237] T 00000000000000000000000000000000 P d6c270b387b04f17b2a58bfcb0eb4f82 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 11, Committed index: 11, Last appended: 1.11, Last appended by leader: 11, Current term: 3, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0db1d539946942f794b0c4ec81f8f4b4" member_type: VOTER last_known_addr { host: "127.3.46.62" port: 46071 } } peers { permanent_uuid: "d6c270b387b04f17b2a58bfcb0eb4f82" member_type: VOTER last_known_addr { host: "127.3.46.61" port: 39407 } } peers { permanent_uuid: "b1731f8308c145f686f88efd6a54b851" member_type: VOTER last_known_addr { host: "127.3.46.60" port: 36127 } }
20:27:43.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:43.088822 4089 sys_catalog.cc:455] T 00000000000000000000000000000000 P d6c270b387b04f17b2a58bfcb0eb4f82 [sys.catalog]: SysCatalogTable state changed. Reason: New leader d6c270b387b04f17b2a58bfcb0eb4f82. Latest consensus state: current_term: 3 leader_uuid: "d6c270b387b04f17b2a58bfcb0eb4f82" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0db1d539946942f794b0c4ec81f8f4b4" member_type: VOTER last_known_addr { host: "127.3.46.62" port: 46071 } } peers { permanent_uuid: "d6c270b387b04f17b2a58bfcb0eb4f82" member_type: VOTER last_known_addr { host: "127.3.46.61" port: 39407 } } peers { permanent_uuid: "b1731f8308c145f686f88efd6a54b851" member_type: VOTER last_known_addr { host: "127.3.46.60" port: 36127 } } }
20:27:43.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:43.089119 4089 sys_catalog.cc:458] T 00000000000000000000000000000000 P d6c270b387b04f17b2a58bfcb0eb4f82 [sys.catalog]: This master's current role is: LEADER
20:27:43.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:43.089848 4090 catalog_manager.cc:1485] Loading table and tablet metadata into memory...
20:27:43.090 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.62:46071(127.3.46.62:46071)] connecting to peer
20:27:43.091 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.62:46071(127.3.46.62:46071): Connection refused: /127.3.46.62:46071
20:27:43.091 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.62:46071(127.3.46.62:46071)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.62:46071(127.3.46.62:46071): Connection refused: /127.3.46.62:46071
20:27:43.091 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.62:46071: Failed to connect to peer master-127.3.46.62:46071(127.3.46.62:46071): Connection refused: /127.3.46.62:46071
20:27:43.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:43.092832 4090 catalog_manager.cc:679] Loaded metadata for table txn_manager_fallback_rolling [id=346e6e6149384e3992393d04f9f16b9c]
20:27:43.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:43.093384 4090 catalog_manager.cc:679] Loaded metadata for table kudu_system.kudu_transactions [id=aca98b500f164adba6b0ae55bd806e83]
20:27:43.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:43.095139 4090 tablet_loader.cc:96] loaded metadata for tablet ce76969b09994fd7b9068b7f105e8fc4 (table kudu_system.kudu_transactions [id=aca98b500f164adba6b0ae55bd806e83])
20:27:43.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:43.095626 4090 tablet_loader.cc:96] loaded metadata for tablet f9ccfdb32e164e61bde68775baab7f9d (table txn_manager_fallback_rolling [id=346e6e6149384e3992393d04f9f16b9c])
20:27:43.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:43.095988 4090 tablet_loader.cc:96] loaded metadata for tablet fe829cba01eb4acb9847b261804f81c8 (table txn_manager_fallback_rolling [id=346e6e6149384e3992393d04f9f16b9c])
20:27:43.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:43.096284 4090 catalog_manager.cc:1494] Initializing Kudu cluster ID...
20:27:43.097 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:43.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:43.097498 4090 catalog_manager.cc:1269] Loaded cluster ID: 3170e931b69f4506aaadd4d1dc1b2458
20:27:43.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:43.097626 4090 catalog_manager.cc:1505] Initializing Kudu internal certificate authority...
20:27:43.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:43.099130 4090 catalog_manager.cc:1514] Loading token signing keys...
20:27:43.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:43.100013 4090 catalog_manager.cc:6055] T 00000000000000000000000000000000 P d6c270b387b04f17b2a58bfcb0eb4f82: Loaded TSK: 0
20:27:43.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:43.101173 4090 catalog_manager.cc:1524] Initializing in-progress tserver states...
20:27:43.110 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.62:46071(127.3.46.62:46071)] connecting to peer
20:27:43.111 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.62:46071(127.3.46.62:46071): Connection refused: /127.3.46.62:46071
20:27:43.112 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.62:46071(127.3.46.62:46071)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.62:46071(127.3.46.62:46071): Connection refused: /127.3.46.62:46071
20:27:43.112 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.62:46071: Failed to connect to peer master-127.3.46.62:46071(127.3.46.62:46071): Connection refused: /127.3.46.62:46071
20:27:43.115 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
20:27:43.115 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=4, TimeoutTracker(timeout=250, elapsed=64), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: Network error: Failed to connect to peer master-127.3.46.62:46071(127.3.46.62:46071): Connection refused: /127.3.46.62:46071, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: OK, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.62:46071(127.3.46.62:46071): Connection refused: /127.3.46.62:46071, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: Network error: Failed to connect to peer master-127.3.46.62:46071(127.3.46.62:46071): Connection refused: /127.3.46.62:46071, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: OK, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.62:46071(127.3.46.62:46071): Connection refused: /127.3.46.62:46071, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [40ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: Network error: Failed to connect to peer master-127.3.46.62:46071(127.3.46.62:46071): Connection refused: /127.3.46.62:46071, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: OK, [46ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: OK, [46ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.62:46071(127.3.46.62:46071): Connection refused: /127.3.46.62:46071, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: Network error: Failed to connect to peer master-127.3.46.62:46071(127.3.46.62:46071): Connection refused: /127.3.46.62:46071, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: OK, [63ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: OK, deferred=Deferred@1435236757(state=PENDING, result=null, callback=(continuation of Deferred@1251649619 after retry RPC after error@76193837) -> (continuation of Deferred@1150610982 after retry RPC after error@1314517104) -> (continuation of Deferred@1369309656 after retry RPC after error@12867598), errback=(continuation of Deferred@1251649619 after retry RPC after error@76193837) -> (continuation of Deferred@1150610982 after retry RPC after error@1314517104) -> (continuation of Deferred@1369309656 after retry RPC after error@12867598))) after lookup
20:27:43.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:43.173785 3379 catalog_manager.cc:2257] Servicing CreateTable request from {username='slave'} at 127.0.0.1:44616:
20:27:43.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "kudu_system.kudu_transactions"
20:27:43.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
20:27:43.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
20:27:43.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
20:27:43.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT64
20:27:43.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
20:27:43.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
20:27:43.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
20:27:43.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
20:27:43.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
20:27:43.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
20:27:43.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
20:27:43.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
20:27:43.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "entry_type"
20:27:43.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT8
20:27:43.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
20:27:43.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
20:27:43.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
20:27:43.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
20:27:43.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
20:27:43.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
20:27:43.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
20:27:43.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
20:27:43.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "identifier"
20:27:43.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
20:27:43.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
20:27:43.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
20:27:43.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
20:27:43.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
20:27:43.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
20:27:43.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
20:27:43.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
20:27:43.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
20:27:43.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "metadata"
20:27:43.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
20:27:43.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
20:27:43.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
20:27:43.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
20:27:43.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
20:27:43.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
20:27:43.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
20:27:43.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
20:27:43.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
20:27:43.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_replicas: 3
20:27:43.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) split_rows_range_bounds {
20:27:43.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) rows: "<redacted>""\006\001\000\000\000\000\000\000\000\000\007\001@B\017\000\000\000\000\000"
20:27:43.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) indirect_data: "<redacted>"""
20:27:43.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
20:27:43.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
20:27:43.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) range_schema {
20:27:43.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
20:27:43.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
20:27:43.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
20:27:43.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
20:27:43.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
20:27:43.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) table_type: TXN_STATUS_TABLE
20:27:43.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:27:43.178956 3379 catalog_manager.cc:7033] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table kudu_system.kudu_transactions in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
20:27:43.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:27:43.211232 3364 consensus_peers.cc:597] T 00000000000000000000000000000000 P d6c270b387b04f17b2a58bfcb0eb4f82 -> Peer 0db1d539946942f794b0c4ec81f8f4b4 (127.3.46.62:46071): Couldn't send request to peer 0db1d539946942f794b0c4ec81f8f4b4. Status: Network error: Client connection negotiation failed: client connection to 127.3.46.62:46071: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
20:27:43.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:43.213857 3469 raft_consensus.cc:1275] T 00000000000000000000000000000000 P b1731f8308c145f686f88efd6a54b851 [term 3 FOLLOWER]: Refusing update from remote peer d6c270b387b04f17b2a58bfcb0eb4f82: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 3 index: 12. (index mismatch)
20:27:43.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:43.214556 4089 consensus_queue.cc:1048] T 00000000000000000000000000000000 P d6c270b387b04f17b2a58bfcb0eb4f82 [LEADER]: Connected to new peer: Peer: permanent_uuid: "b1731f8308c145f686f88efd6a54b851" member_type: VOTER last_known_addr { host: "127.3.46.60" port: 36127 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
20:27:43.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:43.216429 4078 sys_catalog.cc:455] T 00000000000000000000000000000000 P b1731f8308c145f686f88efd6a54b851 [sys.catalog]: SysCatalogTable state changed. Reason: New leader d6c270b387b04f17b2a58bfcb0eb4f82. Latest consensus state: current_term: 3 leader_uuid: "d6c270b387b04f17b2a58bfcb0eb4f82" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0db1d539946942f794b0c4ec81f8f4b4" member_type: VOTER last_known_addr { host: "127.3.46.62" port: 46071 } } peers { permanent_uuid: "d6c270b387b04f17b2a58bfcb0eb4f82" member_type: VOTER last_known_addr { host: "127.3.46.61" port: 39407 } } peers { permanent_uuid: "b1731f8308c145f686f88efd6a54b851" member_type: VOTER last_known_addr { host: "127.3.46.60" port: 36127 } } }
20:27:43.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:43.216722 4078 sys_catalog.cc:458] T 00000000000000000000000000000000 P b1731f8308c145f686f88efd6a54b851 [sys.catalog]: This master's current role is: FOLLOWER
20:27:43.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:43.218216 4079 sys_catalog.cc:455] T 00000000000000000000000000000000 P d6c270b387b04f17b2a58bfcb0eb4f82 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 3 leader_uuid: "d6c270b387b04f17b2a58bfcb0eb4f82" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0db1d539946942f794b0c4ec81f8f4b4" member_type: VOTER last_known_addr { host: "127.3.46.62" port: 46071 } } peers { permanent_uuid: "d6c270b387b04f17b2a58bfcb0eb4f82" member_type: VOTER last_known_addr { host: "127.3.46.61" port: 39407 } } peers { permanent_uuid: "b1731f8308c145f686f88efd6a54b851" member_type: VOTER last_known_addr { host: "127.3.46.60" port: 36127 } } }
20:27:43.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:43.218518 4079 sys_catalog.cc:458] T 00000000000000000000000000000000 P d6c270b387b04f17b2a58bfcb0eb4f82 [sys.catalog]: This master's current role is: LEADER
20:27:43.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:43.219007 4078 sys_catalog.cc:455] T 00000000000000000000000000000000 P b1731f8308c145f686f88efd6a54b851 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 3 leader_uuid: "d6c270b387b04f17b2a58bfcb0eb4f82" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0db1d539946942f794b0c4ec81f8f4b4" member_type: VOTER last_known_addr { host: "127.3.46.62" port: 46071 } } peers { permanent_uuid: "d6c270b387b04f17b2a58bfcb0eb4f82" member_type: VOTER last_known_addr { host: "127.3.46.61" port: 39407 } } peers { permanent_uuid: "b1731f8308c145f686f88efd6a54b851" member_type: VOTER last_known_addr { host: "127.3.46.60" port: 36127 } } }
20:27:43.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:43.219214 4078 sys_catalog.cc:458] T 00000000000000000000000000000000 P b1731f8308c145f686f88efd6a54b851 [sys.catalog]: This master's current role is: FOLLOWER
20:27:43.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:27:43.261324 3849 heartbeater.cc:646] Failed to heartbeat to 127.3.46.62:46071 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.3.46.62:46071: connect: Connection refused (error 111)
20:27:43.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:27:43.270993 3991 heartbeater.cc:646] Failed to heartbeat to 127.3.46.62:46071 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.3.46.62:46071: connect: Connection refused (error 111)
20:27:43.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:27:43.688603 3364 consensus_peers.cc:597] T 00000000000000000000000000000000 P d6c270b387b04f17b2a58bfcb0eb4f82 -> Peer 0db1d539946942f794b0c4ec81f8f4b4 (127.3.46.62:46071): Couldn't send request to peer 0db1d539946942f794b0c4ec81f8f4b4. Status: Network error: Client connection negotiation failed: client connection to 127.3.46.62:46071: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
20:27:43.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:27:43.882302 3364 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.3.46.62:46071: connect: Connection refused (error 111) [suppressed 15 similar messages]
20:27:44.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:27:44.190055 3364 consensus_peers.cc:597] T 00000000000000000000000000000000 P d6c270b387b04f17b2a58bfcb0eb4f82 -> Peer 0db1d539946942f794b0c4ec81f8f4b4 (127.3.46.62:46071): Couldn't send request to peer 0db1d539946942f794b0c4ec81f8f4b4. Status: Network error: Client connection negotiation failed: client connection to 127.3.46.62:46071: connect: Connection refused (error 111). This is attempt 11: this message will repeat every 5th retry.
20:27:44.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:44.262722 3850 heartbeater.cc:499] Master 127.3.46.61:39407 was elected leader, sending a full tablet report...
20:27:44.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:44.269295 3992 heartbeater.cc:499] Master 127.3.46.61:39407 was elected leader, sending a full tablet report...
20:27:44.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:44.369313 3426 master_runner.cc:186] Error getting master registration for 127.3.46.62:46071: Network error: Client connection negotiation failed: client connection to 127.3.46.62:46071: connect: Connection refused (error 111),
20:27:44.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:44.369516 3426 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
20:27:44.467 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet f9ccfdb32e164e61bde68775baab7f9d for table 'txn_manager_fallback_rolling' with partition [0x00000000, 0x00000001)
20:27:44.467 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet fe829cba01eb4acb9847b261804f81c8 for table 'txn_manager_fallback_rolling' with partition [0x00000001, 0x00000002)
20:27:44.467 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x00000000, upperBoundPartitionKey=0x00000001, ttl=299999, tablet=f9ccfdb32e164e61bde68775baab7f9d@[40c77f37a81a464ba7ff42f17bd32c06(127.3.46.2:37949),d00c04330aef4a87a22435384a1a5c12(127.3.46.1:37067)[L],e149de1a3f47443487db92e7c70143dd(127.3.46.3:42083)]}, Tablet{lowerBoundPartitionKey=0x00000001, upperBoundPartitionKey=0x00000002, ttl=299999, tablet=fe829cba01eb4acb9847b261804f81c8@[40c77f37a81a464ba7ff42f17bd32c06(127.3.46.2:37949),d00c04330aef4a87a22435384a1a5c12(127.3.46.1:37067)[L],e149de1a3f47443487db92e7c70143dd(127.3.46.3:42083)]}, NonCoveredRange{lowerBoundPartitionKey=0x00000002, upperBoundPartitionKey=0x, ttl=299999}]
20:27:44.467 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=null, attempt=1, sequence_id=-1, TimeoutTracker(timeout=30000, elapsed=2), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC GetTableLocations: sending RPC to server master-127.3.46.61:39407, [2ms] Sub RPC GetTableLocations: received response from server master-127.3.46.61:39407: OK, deferred=null) after lookup
20:27:44.468 [DEBUG - kudu-nio-1] (Connection.java:748) [peer d00c04330aef4a87a22435384a1a5c12(127.3.46.1:37067)] connecting to peer
20:27:44.469 [DEBUG - kudu-nio-6] (Connection.java:757) [peer d00c04330aef4a87a22435384a1a5c12(127.3.46.1:37067)] Successfully connected to peer
20:27:44.479 [DEBUG - kudu-nio-6] (Negotiator.java:919) Authenticated connection [id: 0xf9ad1055, L:/127.0.0.1:54976 - R:/127.3.46.1:37067] using TOKEN/null
20:27:44.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:27:44.483896 3624 master_proxy_rpc.cc:203] Re-attempting GetTableSchema request to leader Master (127.3.46.62:46071)
20:27:44.523 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.3.46.62:46071
20:27:44.524 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
20:27:44.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:44.525074 3256 external_mini_cluster.cc:1366] Running /tmp/dist-test-taska94UBC/build/asan/bin/kudu
20:27:44.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taska94UBC/build/asan/bin/kudu
20:27:44.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taska94UBC/test-tmp/mini-kudu-cluster1402564250707774898/master-0/wal
20:27:44.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taska94UBC/test-tmp/mini-kudu-cluster1402564250707774898/master-0/data
20:27:44.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:27:44.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:27:44.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:27:44.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:27:44.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:27:44.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:27:44.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taska94UBC/test-tmp/mini-kudu-cluster1402564250707774898/master-0/logs
20:27:44.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taska94UBC/test-tmp/mini-kudu-cluster1402564250707774898/master-0/data/info.pb
20:27:44.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:27:44.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:27:44.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:27:44.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:27:44.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:27:44.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:27:44.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:27:44.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:27:44.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:27:44.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:27:44.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:27:44.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:27:44.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.46.62:46071
20:27:44.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.46.62
20:27:44.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33105
20:27:44.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.46.20:45141
20:27:44.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:27:44.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:27:44.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:27:44.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:27:44.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127
20:27:44.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:27:44.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33531/jwks.json
20:27:44.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled
20:27:44.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100 with env {}
20:27:44.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:27:44.683729 3364 consensus_peers.cc:597] T 00000000000000000000000000000000 P d6c270b387b04f17b2a58bfcb0eb4f82 -> Peer 0db1d539946942f794b0c4ec81f8f4b4 (127.3.46.62:46071): Couldn't send request to peer 0db1d539946942f794b0c4ec81f8f4b4. Status: Network error: Client connection negotiation failed: client connection to 127.3.46.62:46071: connect: Connection refused (error 111). This is attempt 16: this message will repeat every 5th retry.
20:27:44.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:44.833895 3708 heartbeater.cc:499] Master 127.3.46.61:39407 was elected leader, sending a full tablet report...
20:27:44.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:27:44.880801 4132 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
20:27:44.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:27:44.881142 4132 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:27:44.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:27:44.881239 4132 flags.cc:432] Enabled unsafe flag: --never_fsync=true
20:27:44.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:27:44.890906 4132 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
20:27:44.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:27:44.891018 4132 flags.cc:432] Enabled experimental flag: --txn_manager_enabled=true
20:27:44.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:27:44.891065 4132 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
20:27:44.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:27:44.891111 4132 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
20:27:44.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:27:44.891165 4132 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
20:27:44.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:27:44.891202 4132 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
20:27:44.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:27:44.891244 4132 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:33531/jwks.json
20:27:44.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:44.902531 4132 master_runner.cc:386] Master server non-default flags:
20:27:44.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:27:44.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.46.20:45141
20:27:44.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:27:44.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:27:44.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100
20:27:44.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taska94UBC/test-tmp/mini-kudu-cluster1402564250707774898/master-0/data
20:27:44.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taska94UBC/test-tmp/mini-kudu-cluster1402564250707774898/master-0/wal
20:27:44.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:27:44.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127
20:27:44.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled=true
20:27:44.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:27:44.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:27:44.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:27:44.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.46.62:46071
20:27:44.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:27:44.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:27:44.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:27:44.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33531/jwks.json
20:27:44.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:27:44.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:27:44.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taska94UBC/test-tmp/mini-kudu-cluster1402564250707774898/master-0/data/info.pb
20:27:44.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.46.62
20:27:44.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33105
20:27:44.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:27:44.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:27:44.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:27:44.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:27:44.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:27:44.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taska94UBC/test-tmp/mini-kudu-cluster1402564250707774898/master-0/logs
20:27:44.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:27:44.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:27:44.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:27:44.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:27:44.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
20:27:44.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0ba7f54e15c9f673bf3e350f30b8d5f1a4037b79
20:27:44.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:27:44.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 01 May 2026 19:49:29 UTC on bdcb31816ec0
20:27:44.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11698
20:27:44.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
20:27:44.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:44.904137 4132 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:27:44.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:44.905973 4132 file_cache.cc:492] Constructed file cache file cache with capacity 419430
20:27:44.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:27:44.914579 4139 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:27:44.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:27:44.914752 4138 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:27:44.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:27:44.914984 4141 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:27:44.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:44.915833 4132 server_base.cc:1061] running on GCE node
20:27:44.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:44.916640 4132 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:27:44.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:44.918201 4132 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:27:44.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:44.919445 4132 hybrid_clock.cc:648] HybridClock initialized: now 1777667264919352 us; error 67 us; skew 500 ppm
20:27:44.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:44.919786 4132 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:27:44.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:44.922279 4132 webserver.cc:492] Webserver started at http://127.3.46.62:33105/ using document root <none> and password file <none>
20:27:44.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:44.923138 4132 fs_manager.cc:362] Metadata directory not provided
20:27:44.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:44.923290 4132 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:27:44.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:44.929162 4132 fs_manager.cc:714] Time spent opening directory manager: real 0.004s user 0.002s sys 0.004s
20:27:44.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:44.932325 4147 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:27:44.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:44.934111 4132 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.001s sys 0.002s
20:27:44.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:44.934285 4132 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taska94UBC/test-tmp/mini-kudu-cluster1402564250707774898/master-0/data,/tmp/dist-test-taska94UBC/test-tmp/mini-kudu-cluster1402564250707774898/master-0/wal
20:27:44.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0db1d539946942f794b0c4ec81f8f4b4"
20:27:44.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-01 20:27:38 on dist-test-slave-f471"
20:27:44.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:44.935046 4132 fs_report.cc:389] FS layout report
20:27:44.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:27:44.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taska94UBC/test-tmp/mini-kudu-cluster1402564250707774898/master-0/wal
20:27:44.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taska94UBC/test-tmp/mini-kudu-cluster1402564250707774898/master-0/wal
20:27:44.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taska94UBC/test-tmp/mini-kudu-cluster1402564250707774898/master-0/data/data
20:27:44.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:27:44.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:27:44.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:27:44.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:27:44.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:27:44.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:27:44.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:27:44.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:27:44.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:27:44.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:27:44.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:27:44.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:44.959837 4132 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
20:27:44.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:44.960731 4132 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:27:44.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:44.960987 4132 kserver.cc:163] Server-wide thread pool size limit: 3276
20:27:44.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:44.987690 4132 rpc_server.cc:307] RPC server started. Bound to: 127.3.46.62:46071
20:27:44.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:44.987723 4208 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.3.46.62:46071 every 8 connection(s)
20:27:44.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:44.989346 4132 server_base.cc:1193] Dumped server information to /tmp/dist-test-taska94UBC/test-tmp/mini-kudu-cluster1402564250707774898/master-0/data/info.pb
20:27:44.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:44.994483 3256 external_mini_cluster.cc:1428] Started /tmp/dist-test-taska94UBC/build/asan/bin/kudu as pid 4132
20:27:44.994 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:27:44.995 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.3.46.61:39407
20:27:44.995 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
20:27:44.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:44.995504 3256 external_mini_cluster.cc:1699] Attempting to check leaks for /tmp/dist-test-taska94UBC/build/asan/bin/kudu pid 3347
20:27:44.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:44.996662 4209 sys_catalog.cc:263] Verifying existing consensus state
20:27:44.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:44.996695 3419 generic_service.cc:196] Checking for leaks (request via RPC)
20:27:45.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:45.000854 4209 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 0db1d539946942f794b0c4ec81f8f4b4: Bootstrap starting.
20:27:45.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:45.019662 4209 log.cc:826] T 00000000000000000000000000000000 P 0db1d539946942f794b0c4ec81f8f4b4: Log is configured to *not* fsync() on all Append() calls
20:27:45.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:45.036661 4209 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 0db1d539946942f794b0c4ec81f8f4b4: Bootstrap replayed 1/1 log segments. Stats: ops{read=11 overwritten=0 applied=11 ignored=0} inserts{seen=8 ignored=0} mutations{seen=6 ignored=0} orphaned_commits=0. Pending: 0 replicates
20:27:45.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:45.037302 4209 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 0db1d539946942f794b0c4ec81f8f4b4: Bootstrap complete.
20:27:45.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:45.044600 4209 raft_consensus.cc:359] T 00000000000000000000000000000000 P 0db1d539946942f794b0c4ec81f8f4b4 [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0db1d539946942f794b0c4ec81f8f4b4" member_type: VOTER last_known_addr { host: "127.3.46.62" port: 46071 } } peers { permanent_uuid: "d6c270b387b04f17b2a58bfcb0eb4f82" member_type: VOTER last_known_addr { host: "127.3.46.61" port: 39407 } } peers { permanent_uuid: "b1731f8308c145f686f88efd6a54b851" member_type: VOTER last_known_addr { host: "127.3.46.60" port: 36127 } }
20:27:45.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:45.045370 4209 raft_consensus.cc:740] T 00000000000000000000000000000000 P 0db1d539946942f794b0c4ec81f8f4b4 [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: 0db1d539946942f794b0c4ec81f8f4b4, State: Initialized, Role: FOLLOWER
20:27:45.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:45.046164 4209 consensus_queue.cc:260] T 00000000000000000000000000000000 P 0db1d539946942f794b0c4ec81f8f4b4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 11, Last appended: 1.11, Last appended by leader: 11, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0db1d539946942f794b0c4ec81f8f4b4" member_type: VOTER last_known_addr { host: "127.3.46.62" port: 46071 } } peers { permanent_uuid: "d6c270b387b04f17b2a58bfcb0eb4f82" member_type: VOTER last_known_addr { host: "127.3.46.61" port: 39407 } } peers { permanent_uuid: "b1731f8308c145f686f88efd6a54b851" member_type: VOTER last_known_addr { host: "127.3.46.60" port: 36127 } }
20:27:45.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:45.047075 4213 sys_catalog.cc:455] T 00000000000000000000000000000000 P 0db1d539946942f794b0c4ec81f8f4b4 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0db1d539946942f794b0c4ec81f8f4b4" member_type: VOTER last_known_addr { host: "127.3.46.62" port: 46071 } } peers { permanent_uuid: "d6c270b387b04f17b2a58bfcb0eb4f82" member_type: VOTER last_known_addr { host: "127.3.46.61" port: 39407 } } peers { permanent_uuid: "b1731f8308c145f686f88efd6a54b851" member_type: VOTER last_known_addr { host: "127.3.46.60" port: 36127 } } }
20:27:45.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:45.047416 4213 sys_catalog.cc:458] T 00000000000000000000000000000000 P 0db1d539946942f794b0c4ec81f8f4b4 [sys.catalog]: This master's current role is: FOLLOWER
20:27:45.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:45.048395 4209 sys_catalog.cc:565] T 00000000000000000000000000000000 P 0db1d539946942f794b0c4ec81f8f4b4 [sys.catalog]: configured and running, proceeding with master startup.
20:27:45.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:45.060041 4224 catalog_manager.cc:1269] Loaded cluster ID: 3170e931b69f4506aaadd4d1dc1b2458
20:27:45.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:45.060178 4224 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 0db1d539946942f794b0c4ec81f8f4b4: loading cluster ID for follower catalog manager: success
20:27:45.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:45.064965 4224 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 0db1d539946942f794b0c4ec81f8f4b4: acquiring CA information for follower catalog manager: success
20:27:45.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:45.073277 4224 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 0db1d539946942f794b0c4ec81f8f4b4: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
20:27:45.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:45.152007 3256 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taska94UBC/build/asan/bin/kudu with pid 3347
20:27:45.174 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:27:45.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:27:45.175128 4226 negotiation.cc:336] Failed RPC negotiation. Trace:
20:27:45.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0501 20:27:45.082467 (+ 0us) reactor.cc:678] Submitting negotiation task for client connection to 127.3.46.61:39407 (local address 127.0.0.1:44618)
20:27:45.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0501 20:27:45.082635 (+ 168us) negotiation.cc:107] Waiting for socket to connect
20:27:45.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0501 20:27:45.082651 (+ 16us) client_negotiation.cc:175] Beginning negotiation
20:27:45.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0501 20:27:45.082745 (+ 94us) client_negotiation.cc:253] Sending NEGOTIATE NegotiatePB request
20:27:45.175 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.3.46.61:39407(127.3.46.61:39407)] handling channelInactive
20:27:45.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0501 20:27:45.173274 (+ 90529us) negotiation.cc:326] Negotiation complete: Network error: Client connection negotiation failed: client connection to 127.3.46.61:39407: BlockingRecv error: recv error from unknown peer: Transport endpoint is not connected (error 107)
20:27:45.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"client-negotiator.queue_time_us":57}
20:27:45.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:27:45.175400 3445 connection.cc:570] server connection from 127.0.0.1:54932 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
20:27:45.175 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state READY due to: connection closed
20:27:45.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:45.175906 4132 master_runner.cc:186] Error getting master registration for 127.3.46.61:39407: Network error: Client connection negotiation failed: client connection to 127.3.46.61:39407: BlockingRecv error: recv error from unknown peer: Transport endpoint is not connected (error 107),
20:27:45.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:45.182647 4132 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
20:27:45.271 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:45.272 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:45.272 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:45.272 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location master-127.3.46.61:39407(127.3.46.61:39407) for tablet Kudu Master: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:45.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:45.272276 3849 heartbeater.cc:344] Connected to a master server at 127.3.46.62:46071
20:27:45.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:45.275580 4164 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "40c77f37a81a464ba7ff42f17bd32c06" instance_seqno: 1777667260738894) as {username='slave'} at 127.3.46.2:48123; Asking this server to re-register.
20:27:45.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:45.277522 3849 heartbeater.cc:461] Registering TS with master...
20:27:45.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:45.280769 4164 ts_manager.cc:194] Registered new tserver with Master: 40c77f37a81a464ba7ff42f17bd32c06 (127.3.46.2:37949)
20:27:45.291 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.62:46071(127.3.46.62:46071)] connecting to peer
20:27:45.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:45.290961 3991 heartbeater.cc:344] Connected to a master server at 127.3.46.62:46071
20:27:45.292 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:45.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:45.292232 4213 raft_consensus.cc:493] T 00000000000000000000000000000000 P 0db1d539946942f794b0c4ec81f8f4b4 [term 1 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
20:27:45.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:45.292410 4164 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "e149de1a3f47443487db92e7c70143dd" instance_seqno: 1777667261304872) as {username='slave'} at 127.3.46.3:51405; Asking this server to re-register.
20:27:45.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:45.292508 4213 raft_consensus.cc:515] T 00000000000000000000000000000000 P 0db1d539946942f794b0c4ec81f8f4b4 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0db1d539946942f794b0c4ec81f8f4b4" member_type: VOTER last_known_addr { host: "127.3.46.62" port: 46071 } } peers { permanent_uuid: "d6c270b387b04f17b2a58bfcb0eb4f82" member_type: VOTER last_known_addr { host: "127.3.46.61" port: 39407 } } peers { permanent_uuid: "b1731f8308c145f686f88efd6a54b851" member_type: VOTER last_known_addr { host: "127.3.46.60" port: 36127 } }
20:27:45.292 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.3.46.62:46071(127.3.46.62:46071)] Successfully connected to peer
20:27:45.293 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:45.293 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:45.293 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:45.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:45.293468 3991 heartbeater.cc:461] Registering TS with master...
20:27:45.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:45.294986 4164 ts_manager.cc:194] Registered new tserver with Master: e149de1a3f47443487db92e7c70143dd (127.3.46.3:42083)
20:27:45.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:45.294967 4213 leader_election.cc:290] T 00000000000000000000000000000000 P 0db1d539946942f794b0c4ec81f8f4b4 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers d6c270b387b04f17b2a58bfcb0eb4f82 (127.3.46.61:39407), b1731f8308c145f686f88efd6a54b851 (127.3.46.60:36127)
20:27:45.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:27:45.296902 4149 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.3.46.61:39407: connect: Connection refused (error 111)
20:27:45.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:27:45.331075 4149 leader_election.cc:336] T 00000000000000000000000000000000 P 0db1d539946942f794b0c4ec81f8f4b4 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer d6c270b387b04f17b2a58bfcb0eb4f82 (127.3.46.61:39407): Network error: Client connection negotiation failed: client connection to 127.3.46.61:39407: connect: Connection refused (error 111)
20:27:45.332 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0xede89606, L:/127.0.0.1:48180 - R:/127.3.46.62:46071] using TOKEN/null
20:27:45.334 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:45.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:45.338944 3469 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "0db1d539946942f794b0c4ec81f8f4b4" candidate_term: 2 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "b1731f8308c145f686f88efd6a54b851" is_pre_election: true
20:27:45.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:45.340629 4148 leader_election.cc:304] T 00000000000000000000000000000000 P 0db1d539946942f794b0c4ec81f8f4b4 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 0db1d539946942f794b0c4ec81f8f4b4; no voters: b1731f8308c145f686f88efd6a54b851, d6c270b387b04f17b2a58bfcb0eb4f82
20:27:45.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:45.341913 4213 raft_consensus.cc:2749] T 00000000000000000000000000000000 P 0db1d539946942f794b0c4ec81f8f4b4 [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
20:27:45.351 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:45.352 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:45.352 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:45.353 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:45.354 [WARN - kudu-nio-2] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:45.371 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:45.372 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:45.372 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:45.373 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:45.373 [WARN - kudu-nio-2] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:45.390 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:45.392 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:45.392 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:45.392 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:45.394 [WARN - kudu-nio-2] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:45.431 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:45.432 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:45.432 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:45.432 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:45.433 [WARN - kudu-nio-2] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:45.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:45.454022 4236 raft_consensus.cc:493] T 00000000000000000000000000000000 P b1731f8308c145f686f88efd6a54b851 [term 3 FOLLOWER]: Starting pre-election (detected failure of leader d6c270b387b04f17b2a58bfcb0eb4f82)
20:27:45.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:45.454218 4236 raft_consensus.cc:515] T 00000000000000000000000000000000 P b1731f8308c145f686f88efd6a54b851 [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0db1d539946942f794b0c4ec81f8f4b4" member_type: VOTER last_known_addr { host: "127.3.46.62" port: 46071 } } peers { permanent_uuid: "d6c270b387b04f17b2a58bfcb0eb4f82" member_type: VOTER last_known_addr { host: "127.3.46.61" port: 39407 } } peers { permanent_uuid: "b1731f8308c145f686f88efd6a54b851" member_type: VOTER last_known_addr { host: "127.3.46.60" port: 36127 } }
20:27:45.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:45.454881 4236 leader_election.cc:290] T 00000000000000000000000000000000 P b1731f8308c145f686f88efd6a54b851 [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 0db1d539946942f794b0c4ec81f8f4b4 (127.3.46.62:46071), d6c270b387b04f17b2a58bfcb0eb4f82 (127.3.46.61:39407)
20:27:45.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:27:45.458690 3445 leader_election.cc:336] T 00000000000000000000000000000000 P b1731f8308c145f686f88efd6a54b851 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer d6c270b387b04f17b2a58bfcb0eb4f82 (127.3.46.61:39407): Network error: Client connection negotiation failed: client connection to 127.3.46.61:39407: connect: Connection refused (error 111)
20:27:45.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:45.463011 4174 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "b1731f8308c145f686f88efd6a54b851" candidate_term: 4 candidate_status { last_received { term: 3 index: 12 } } ignore_live_leader: false dest_uuid: "0db1d539946942f794b0c4ec81f8f4b4" is_pre_election: true
20:27:45.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:45.463523 4174 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 0db1d539946942f794b0c4ec81f8f4b4 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate b1731f8308c145f686f88efd6a54b851 in term 1.
20:27:45.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:45.464084 3445 leader_election.cc:304] T 00000000000000000000000000000000 P b1731f8308c145f686f88efd6a54b851 [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 0db1d539946942f794b0c4ec81f8f4b4, b1731f8308c145f686f88efd6a54b851; no voters: d6c270b387b04f17b2a58bfcb0eb4f82
20:27:45.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:45.464448 4236 raft_consensus.cc:2804] T 00000000000000000000000000000000 P b1731f8308c145f686f88efd6a54b851 [term 3 FOLLOWER]: Leader pre-election won for term 4
20:27:45.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:45.464591 4236 raft_consensus.cc:493] T 00000000000000000000000000000000 P b1731f8308c145f686f88efd6a54b851 [term 3 FOLLOWER]: Starting leader election (detected failure of leader d6c270b387b04f17b2a58bfcb0eb4f82)
20:27:45.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:45.464691 4236 raft_consensus.cc:3060] T 00000000000000000000000000000000 P b1731f8308c145f686f88efd6a54b851 [term 3 FOLLOWER]: Advancing to term 4
20:27:45.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:45.467665 4236 raft_consensus.cc:515] T 00000000000000000000000000000000 P b1731f8308c145f686f88efd6a54b851 [term 4 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0db1d539946942f794b0c4ec81f8f4b4" member_type: VOTER last_known_addr { host: "127.3.46.62" port: 46071 } } peers { permanent_uuid: "d6c270b387b04f17b2a58bfcb0eb4f82" member_type: VOTER last_known_addr { host: "127.3.46.61" port: 39407 } } peers { permanent_uuid: "b1731f8308c145f686f88efd6a54b851" member_type: VOTER last_known_addr { host: "127.3.46.60" port: 36127 } }
20:27:45.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:45.468462 4236 leader_election.cc:290] T 00000000000000000000000000000000 P b1731f8308c145f686f88efd6a54b851 [CANDIDATE]: Term 4 election: Requested vote from peers 0db1d539946942f794b0c4ec81f8f4b4 (127.3.46.62:46071), d6c270b387b04f17b2a58bfcb0eb4f82 (127.3.46.61:39407)
20:27:45.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:45.469022 4174 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "b1731f8308c145f686f88efd6a54b851" candidate_term: 4 candidate_status { last_received { term: 3 index: 12 } } ignore_live_leader: false dest_uuid: "0db1d539946942f794b0c4ec81f8f4b4"
20:27:45.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:45.469257 4174 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 0db1d539946942f794b0c4ec81f8f4b4 [term 1 FOLLOWER]: Advancing to term 4
20:27:45.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:27:45.470005 3445 leader_election.cc:336] T 00000000000000000000000000000000 P b1731f8308c145f686f88efd6a54b851 [CANDIDATE]: Term 4 election: RPC error from VoteRequest() call to peer d6c270b387b04f17b2a58bfcb0eb4f82 (127.3.46.61:39407): Network error: Client connection negotiation failed: client connection to 127.3.46.61:39407: connect: Connection refused (error 111)
20:27:45.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:45.475095 4174 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 0db1d539946942f794b0c4ec81f8f4b4 [term 4 FOLLOWER]: Leader election vote request: Granting yes vote for candidate b1731f8308c145f686f88efd6a54b851 in term 4.
20:27:45.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:45.475649 3445 leader_election.cc:304] T 00000000000000000000000000000000 P b1731f8308c145f686f88efd6a54b851 [CANDIDATE]: Term 4 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 0db1d539946942f794b0c4ec81f8f4b4, b1731f8308c145f686f88efd6a54b851; no voters: d6c270b387b04f17b2a58bfcb0eb4f82
20:27:45.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:45.476063 4236 raft_consensus.cc:2804] T 00000000000000000000000000000000 P b1731f8308c145f686f88efd6a54b851 [term 4 FOLLOWER]: Leader election won for term 4
20:27:45.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:45.476588 4236 raft_consensus.cc:697] T 00000000000000000000000000000000 P b1731f8308c145f686f88efd6a54b851 [term 4 LEADER]: Becoming Leader. State: Replica: b1731f8308c145f686f88efd6a54b851, State: Running, Role: LEADER
20:27:45.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:45.477159 4236 consensus_queue.cc:237] T 00000000000000000000000000000000 P b1731f8308c145f686f88efd6a54b851 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 12, Committed index: 12, Last appended: 3.12, Last appended by leader: 12, Current term: 4, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0db1d539946942f794b0c4ec81f8f4b4" member_type: VOTER last_known_addr { host: "127.3.46.62" port: 46071 } } peers { permanent_uuid: "d6c270b387b04f17b2a58bfcb0eb4f82" member_type: VOTER last_known_addr { host: "127.3.46.61" port: 39407 } } peers { permanent_uuid: "b1731f8308c145f686f88efd6a54b851" member_type: VOTER last_known_addr { host: "127.3.46.60" port: 36127 } }
20:27:45.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:45.479328 4240 sys_catalog.cc:455] T 00000000000000000000000000000000 P b1731f8308c145f686f88efd6a54b851 [sys.catalog]: SysCatalogTable state changed. Reason: New leader b1731f8308c145f686f88efd6a54b851. Latest consensus state: current_term: 4 leader_uuid: "b1731f8308c145f686f88efd6a54b851" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0db1d539946942f794b0c4ec81f8f4b4" member_type: VOTER last_known_addr { host: "127.3.46.62" port: 46071 } } peers { permanent_uuid: "d6c270b387b04f17b2a58bfcb0eb4f82" member_type: VOTER last_known_addr { host: "127.3.46.61" port: 39407 } } peers { permanent_uuid: "b1731f8308c145f686f88efd6a54b851" member_type: VOTER last_known_addr { host: "127.3.46.60" port: 36127 } } }
20:27:45.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:45.479669 4240 sys_catalog.cc:458] T 00000000000000000000000000000000 P b1731f8308c145f686f88efd6a54b851 [sys.catalog]: This master's current role is: LEADER
20:27:45.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:45.480346 4242 catalog_manager.cc:1485] Loading table and tablet metadata into memory...
20:27:45.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:45.482654 4242 catalog_manager.cc:679] Loaded metadata for table txn_manager_fallback_rolling [id=346e6e6149384e3992393d04f9f16b9c]
20:27:45.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:45.483100 4242 catalog_manager.cc:679] Loaded metadata for table kudu_system.kudu_transactions [id=aca98b500f164adba6b0ae55bd806e83]
20:27:45.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:45.484642 4242 tablet_loader.cc:96] loaded metadata for tablet ce76969b09994fd7b9068b7f105e8fc4 (table kudu_system.kudu_transactions [id=aca98b500f164adba6b0ae55bd806e83])
20:27:45.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:45.485174 4242 tablet_loader.cc:96] loaded metadata for tablet f9ccfdb32e164e61bde68775baab7f9d (table txn_manager_fallback_rolling [id=346e6e6149384e3992393d04f9f16b9c])
20:27:45.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:45.485483 4242 tablet_loader.cc:96] loaded metadata for tablet fe829cba01eb4acb9847b261804f81c8 (table txn_manager_fallback_rolling [id=346e6e6149384e3992393d04f9f16b9c])
20:27:45.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:45.485703 4242 catalog_manager.cc:1494] Initializing Kudu cluster ID...
20:27:45.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:45.486575 4242 catalog_manager.cc:1269] Loaded cluster ID: 3170e931b69f4506aaadd4d1dc1b2458
20:27:45.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:45.486670 4242 catalog_manager.cc:1505] Initializing Kudu internal certificate authority...
20:27:45.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:45.487918 4242 catalog_manager.cc:1514] Loading token signing keys...
20:27:45.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:45.488890 4242 catalog_manager.cc:6055] T 00000000000000000000000000000000 P b1731f8308c145f686f88efd6a54b851: Loaded TSK: 0
20:27:45.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:45.490039 4242 catalog_manager.cc:1524] Initializing in-progress tserver states...
20:27:45.490 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:45.492 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:45.492 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:45.492 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:45.495 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
20:27:45.496 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=7, TimeoutTracker(timeout=250, elapsed=224), Traces: [0ms] sending RPC to server master-127.3.46.61:39407, [1ms] received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] delaying RPC due to: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [27ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: OK, [63ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [83ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [119ms] refreshing cache from master, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [159ms] refreshing cache from master, [159ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [219ms] refreshing cache from master, [219ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [219ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [224ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: OK, deferred=Deferred@798881121(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$233/0x00007f8b98336cd0@569b69de -> (continuation of Deferred@275689571 after retry RPC after error@1797613749) -> (continuation of Deferred@735293005 after retry RPC after error@638318448) -> (continuation of Deferred@1132701306 after retry RPC after error@61021823) -> (continuation of Deferred@892627328 after retry RPC after error@1283307553) -> (continuation of Deferred@303265698 after retry RPC after error@165204857), errback=org.apache.kudu.client.KuduTransaction$$Lambda$234/0x00007f8b98336ef8@1fb6c3ab -> (continuation of Deferred@275689571 after retry RPC after error@1797613749) -> (continuation of Deferred@735293005 after retry RPC after error@638318448) -> (continuation of Deferred@1132701306 after retry RPC after error@61021823) -> (continuation of Deferred@892627328 after retry RPC after error@1283307553) -> (continuation of Deferred@303265698 after retry RPC after error@165204857))) after lookup
20:27:45.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:27:45.530145 3490 rpcz_store.cc:267] Call kudu.transactions.TxnManagerService.KeepTransactionAlive from 127.0.0.1:54942 (request call id 17) took 33 ms (client timeout 25 ms). Trace:
20:27:45.531 [DEBUG - pool-40-thread-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=8, TimeoutTracker(timeout=250, elapsed=259), Traces: [0ms] sending RPC to server master-127.3.46.61:39407, [1ms] received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] delaying RPC due to: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [27ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: OK, [63ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [83ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [119ms] refreshing cache from master, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [159ms] refreshing cache from master, [159ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [219ms] refreshing cache from master, [219ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [219ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [224ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: OK, [225ms] sending RPC to server master-127.3.46.60:36127, deferred=Deferred@798881121(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$233/0x00007f8b98336cd0@569b69de -> (continuation of Deferred@275689571 after retry RPC after error@1797613749) -> (continuation of Deferred@735293005 after retry RPC after error@638318448) -> (continuation of Deferred@1132701306 after retry RPC after error@61021823) -> (continuation of Deferred@892627328 after retry RPC after error@1283307553) -> (continuation of Deferred@303265698 after retry RPC after error@165204857) -> (continuation of Deferred@141610015 after retry RPC@33024537), errback=org.apache.kudu.client.KuduTransaction$$Lambda$234/0x00007f8b98336ef8@1fb6c3ab -> (continuation of Deferred@275689571 after retry RPC after error@1797613749) -> (continuation of Deferred@735293005 after retry RPC after error@638318448) -> (continuation of Deferred@1132701306 after retry RPC after error@61021823) -> (continuation of Deferred@892627328 after retry RPC after error@1283307553) -> (continuation of Deferred@303265698 after retry RPC after error@165204857) -> (continuation of Deferred@141610015 after retry RPC@33024537)))
20:27:45.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:27:45.530488 3490 rpcz_store.cc:269] 0501 20:27:45.497158 (+ 0us) service_pool.cc:167] Inserting onto call queue
20:27:45.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0501 20:27:45.497233 (+ 75us) service_pool.cc:224] Handling call
20:27:45.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0501 20:27:45.530057 (+ 32824us) inbound_call.cc:177] Queueing success response
20:27:45.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"init.queue_time_us":531,"thread_start_us":778,"threads_started":3}
20:27:45.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/monotime.cc:332:28: runtime error: signed integer overflow: 1439332203446 + 9223370597522573101 cannot be represented in type 'long'
20:27:45.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:27:45.577420 3445 consensus_peers.cc:597] T 00000000000000000000000000000000 P b1731f8308c145f686f88efd6a54b851 -> Peer d6c270b387b04f17b2a58bfcb0eb4f82 (127.3.46.61:39407): Couldn't send request to peer d6c270b387b04f17b2a58bfcb0eb4f82. Status: Network error: Client connection negotiation failed: client connection to 127.3.46.61:39407: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
20:27:45.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:45.579689 4174 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 0db1d539946942f794b0c4ec81f8f4b4 [term 4 FOLLOWER]: Refusing update from remote peer b1731f8308c145f686f88efd6a54b851: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 4 index: 13. (index mismatch)
20:27:45.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:45.580843 4240 consensus_queue.cc:1048] T 00000000000000000000000000000000 P b1731f8308c145f686f88efd6a54b851 [LEADER]: Connected to new peer: Peer: permanent_uuid: "0db1d539946942f794b0c4ec81f8f4b4" member_type: VOTER last_known_addr { host: "127.3.46.62" port: 46071 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 11, Time since last communication: 0.000s
20:27:45.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:45.586198 3460 catalog_manager.cc:2257] Servicing CreateTable request from {username='slave'} at 127.0.0.1:54966:
20:27:45.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "kudu_system.kudu_transactions"
20:27:45.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
20:27:45.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
20:27:45.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
20:27:45.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT64
20:27:45.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
20:27:45.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
20:27:45.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
20:27:45.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
20:27:45.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
20:27:45.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
20:27:45.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
20:27:45.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
20:27:45.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "entry_type"
20:27:45.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT8
20:27:45.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
20:27:45.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
20:27:45.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
20:27:45.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
20:27:45.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
20:27:45.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
20:27:45.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
20:27:45.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
20:27:45.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "identifier"
20:27:45.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
20:27:45.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
20:27:45.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
20:27:45.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
20:27:45.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
20:27:45.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
20:27:45.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
20:27:45.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
20:27:45.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
20:27:45.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "metadata"
20:27:45.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
20:27:45.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
20:27:45.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
20:27:45.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
20:27:45.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
20:27:45.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
20:27:45.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
20:27:45.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
20:27:45.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
20:27:45.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_replicas: 3
20:27:45.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) split_rows_range_bounds {
20:27:45.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) rows: "<redacted>""\006\001\000\000\000\000\000\000\000\000\007\001@B\017\000\000\000\000\000"
20:27:45.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) indirect_data: "<redacted>"""
20:27:45.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
20:27:45.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
20:27:45.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) range_schema {
20:27:45.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
20:27:45.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
20:27:45.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
20:27:45.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
20:27:45.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
20:27:45.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) table_type: TXN_STATUS_TABLE
20:27:45.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:27:45.590363 3460 catalog_manager.cc:7033] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table kudu_system.kudu_transactions in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
20:27:45.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:45.595461 4254 sys_catalog.cc:455] T 00000000000000000000000000000000 P 0db1d539946942f794b0c4ec81f8f4b4 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 4 leader_uuid: "b1731f8308c145f686f88efd6a54b851" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0db1d539946942f794b0c4ec81f8f4b4" member_type: VOTER last_known_addr { host: "127.3.46.62" port: 46071 } } peers { permanent_uuid: "d6c270b387b04f17b2a58bfcb0eb4f82" member_type: VOTER last_known_addr { host: "127.3.46.61" port: 39407 } } peers { permanent_uuid: "b1731f8308c145f686f88efd6a54b851" member_type: VOTER last_known_addr { host: "127.3.46.60" port: 36127 } } }
20:27:45.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:45.596127 4254 sys_catalog.cc:458] T 00000000000000000000000000000000 P 0db1d539946942f794b0c4ec81f8f4b4 [sys.catalog]: This master's current role is: FOLLOWER
20:27:45.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:45.596208 4213 sys_catalog.cc:455] T 00000000000000000000000000000000 P 0db1d539946942f794b0c4ec81f8f4b4 [sys.catalog]: SysCatalogTable state changed. Reason: New leader b1731f8308c145f686f88efd6a54b851. Latest consensus state: current_term: 4 leader_uuid: "b1731f8308c145f686f88efd6a54b851" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0db1d539946942f794b0c4ec81f8f4b4" member_type: VOTER last_known_addr { host: "127.3.46.62" port: 46071 } } peers { permanent_uuid: "d6c270b387b04f17b2a58bfcb0eb4f82" member_type: VOTER last_known_addr { host: "127.3.46.61" port: 39407 } } peers { permanent_uuid: "b1731f8308c145f686f88efd6a54b851" member_type: VOTER last_known_addr { host: "127.3.46.60" port: 36127 } } }
20:27:45.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:45.596457 4213 sys_catalog.cc:458] T 00000000000000000000000000000000 P 0db1d539946942f794b0c4ec81f8f4b4 [sys.catalog]: This master's current role is: FOLLOWER
20:27:45.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:45.599583 4213 sys_catalog.cc:455] T 00000000000000000000000000000000 P 0db1d539946942f794b0c4ec81f8f4b4 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 4 leader_uuid: "b1731f8308c145f686f88efd6a54b851" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0db1d539946942f794b0c4ec81f8f4b4" member_type: VOTER last_known_addr { host: "127.3.46.62" port: 46071 } } peers { permanent_uuid: "d6c270b387b04f17b2a58bfcb0eb4f82" member_type: VOTER last_known_addr { host: "127.3.46.61" port: 39407 } } peers { permanent_uuid: "b1731f8308c145f686f88efd6a54b851" member_type: VOTER last_known_addr { host: "127.3.46.60" port: 36127 } } }
20:27:45.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:45.599814 4213 sys_catalog.cc:458] T 00000000000000000000000000000000 P 0db1d539946942f794b0c4ec81f8f4b4 [sys.catalog]: This master's current role is: FOLLOWER
20:27:45.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:45.603899 4236 sys_catalog.cc:455] T 00000000000000000000000000000000 P b1731f8308c145f686f88efd6a54b851 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 4 leader_uuid: "b1731f8308c145f686f88efd6a54b851" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0db1d539946942f794b0c4ec81f8f4b4" member_type: VOTER last_known_addr { host: "127.3.46.62" port: 46071 } } peers { permanent_uuid: "d6c270b387b04f17b2a58bfcb0eb4f82" member_type: VOTER last_known_addr { host: "127.3.46.61" port: 39407 } } peers { permanent_uuid: "b1731f8308c145f686f88efd6a54b851" member_type: VOTER last_known_addr { host: "127.3.46.60" port: 36127 } } }
20:27:45.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:45.604174 4236 sys_catalog.cc:458] T 00000000000000000000000000000000 P b1731f8308c145f686f88efd6a54b851 [sys.catalog]: This master's current role is: LEADER
20:27:45.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:45.843698 3707 heartbeater.cc:344] Connected to a master server at 127.3.46.62:46071
20:27:45.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:45.846216 4164 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "d00c04330aef4a87a22435384a1a5c12" instance_seqno: 1777667260208305) as {username='slave'} at 127.3.46.1:34887; Asking this server to re-register.
20:27:45.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:45.847126 3707 heartbeater.cc:461] Registering TS with master...
20:27:45.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:45.848820 4164 ts_manager.cc:194] Registered new tserver with Master: d00c04330aef4a87a22435384a1a5c12 (127.3.46.1:37067)
20:27:46.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:27:46.079969 3445 consensus_peers.cc:597] T 00000000000000000000000000000000 P b1731f8308c145f686f88efd6a54b851 -> Peer d6c270b387b04f17b2a58bfcb0eb4f82 (127.3.46.61:39407): Couldn't send request to peer d6c270b387b04f17b2a58bfcb0eb4f82. Status: Network error: Client connection negotiation failed: client connection to 127.3.46.61:39407: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
20:27:46.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:46.339236 3990 heartbeater.cc:499] Master 127.3.46.60:36127 was elected leader, sending a full tablet report...
20:27:46.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #0 0x7f23344ae998 in kudu::operator+(kudu::MonoTime const&, kudu::MonoDelta const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/monotime.cc:332:28
20:27:46.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #1 0xa94295 in kudu::CountDownLatch::WaitFor(kudu::MonoDelta const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/countdown_latch.h:99:38
20:27:46.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #2 0x7f23482f98f2 in kudu::Promise<kudu::Status>::WaitFor(kudu::MonoDelta const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/promise.h:57:16
20:27:46.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #3 0x7f23482e9404 in kudu::master::Master::WaitForTxnManagerInit(kudu::MonoTime) const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/master/master.cc:483:48
20:27:46.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:27:46.589130 3445 consensus_peers.cc:597] T 00000000000000000000000000000000 P b1731f8308c145f686f88efd6a54b851 -> Peer d6c270b387b04f17b2a58bfcb0eb4f82 (127.3.46.61:39407): Couldn't send request to peer d6c270b387b04f17b2a58bfcb0eb4f82. Status: Network error: Client connection negotiation failed: client connection to 127.3.46.61:39407: connect: Connection refused (error 111). This is attempt 11: this message will repeat every 5th retry.
20:27:46.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #4 0x7f234844d7ee in kudu::transactions::TxnManager::CheckInitialized(kudu::MonoTime const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/master/txn_manager.cc:283:21
20:27:46.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #5 0x7f234844e3cb in kudu::transactions::TxnManager::KeepTransactionAlive(long, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, kudu::MonoTime const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/master/txn_manager.cc:226:3
20:27:46.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #6 0x7f234845300f in kudu::transactions::TxnManagerServiceImpl::KeepTransactionAlive(kudu::transactions::KeepTransactionAliveRequestPB const*, kudu::transactions::KeepTransactionAliveResponsePB*, kudu::rpc::RpcContext*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/master/txn_manager_service.cc:159:42
20:27:46.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #7 0x7f23422122d6 in kudu::transactions::TxnManagerServiceIf::TxnManagerServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_9::operator()(google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*) const /home/jenkins-slave/workspace/build_and_test_flaky@2/build/asan/src/kudu/master/txn_manager.service.cc:271:13
20:27:46.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #8 0x7f2342212005 in std::_Function_handler<void (google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*), kudu::transactions::TxnManagerServiceIf::TxnManagerServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_9>::_M_invoke(std::_Any_data const&, google::protobuf::Message const*&&, google::protobuf::Message*&&, kudu::rpc::RpcContext*&&) ../../../include/c++/7.5.0/bits/std_function.h:316:2
20:27:46.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #9 0x7f233b54f10e in std::function<void (google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*)>::operator()(google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*) const ../../../include/c++/7.5.0/bits/std_function.h:706:14
20:27:46.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #10 0x7f233b54dfee in kudu::rpc::GeneratedServiceIf::Handle(kudu::rpc::InboundCall*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/rpc/service_if.cc:137:3
20:27:46.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #11 0x7f233b554228 in kudu::rpc::ServicePool::RunThread() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/rpc/service_pool.cc:228:15
20:27:46.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #12 0x7f233b554b8f in kudu::rpc::ServicePool::Init(int)::$_0::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/rpc/service_pool.cc:91:5
20:27:46.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #13 0x7f233b5549b8 in std::_Function_handler<void (), kudu::rpc::ServicePool::Init(int)::$_0>::_M_invoke(std::_Any_data const&) ../../../include/c++/7.5.0/bits/std_function.h:316:2
20:27:46.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #14 0xaa4cac in std::function<void ()>::operator()() const ../../../include/c++/7.5.0/bits/std_function.h:706:14
20:27:46.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #15 0x7f2334581528 in kudu::Thread::SuperviseThread(void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:694:3
20:27:46.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #16 0x7f2337b6f6da in start_thread (/lib/x86_64-linux-gnu/libpthread.so.0+0x76da)
20:27:46.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #17 0x7f233069961e in clone (/lib/x86_64-linux-gnu/libc.so.6+0x12161e)
20:27:46.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:27:46.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) SUMMARY: UndefinedBehaviorSanitizer: undefined-behavior /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/monotime.cc:332:28 in
20:27:46.654 [ERROR - kudu-nio-2] (Connection.java:452) [peer master-127.3.46.60:36127(127.3.46.60:36127)] unexpected exception from downstream on [id: 0x8609ca1e, L:/127.0.0.1:54942 - R:/127.3.46.60:36127]
java.net.SocketException: Connection reset
at java.base/sun.nio.ch.SocketChannelImpl.throwConnectionReset(SocketChannelImpl.java:394)
at java.base/sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:426)
at io.netty.buffer.PooledByteBuf.setBytes(PooledByteBuf.java:255)
at io.netty.buffer.AbstractByteBuf.writeBytes(AbstractByteBuf.java:1132)
at io.netty.channel.socket.nio.NioSocketChannel.doReadBytes(NioSocketChannel.java:356)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:156)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:796)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:732)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:658)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:998)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
at java.base/java.lang.Thread.run(Thread.java:840)
java.net.SocketException: Connection reset
at java.base/sun.nio.ch.SocketChannelImpl.throwConnectionReset(SocketChannelImpl.java:394)
at java.base/sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:426)
at io.netty.buffer.PooledByteBuf.setBytes(PooledByteBuf.java:255)
at io.netty.buffer.AbstractByteBuf.writeBytes(AbstractByteBuf.java:1132)
at io.netty.channel.socket.nio.NioSocketChannel.doReadBytes(NioSocketChannel.java:356)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:156)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:796)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:732)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:658)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:998)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
at java.base/java.lang.Thread.run(Thread.java:840)
20:27:46.654 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state READY due to: [peer master-127.3.46.60:36127(127.3.46.60:36127)] unexpected exception from downstream on [id: 0x8609ca1e, L:/127.0.0.1:54942 - R:/127.3.46.60:36127]
20:27:46.654 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location master-127.3.46.60:36127(127.3.46.60:36127) for tablet Kudu Master: [peer master-127.3.46.60:36127(127.3.46.60:36127)] unexpected exception from downstream on [id: 0x8609ca1e, L:/127.0.0.1:54942 - R:/127.3.46.60:36127]
20:27:46.654 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location master-127.3.46.60:36127(127.3.46.60:36127) for tablet Kudu Master: [peer master-127.3.46.60:36127(127.3.46.60:36127)] unexpected exception from downstream on [id: 0x8609ca1e, L:/127.0.0.1:54942 - R:/127.3.46.60:36127]
20:27:46.655 [DEBUG - kudu-nio-2] (RemoteTablet.java:144) tablet Kudu Master already removed ts master-127.3.46.60:36127, size left is 0
20:27:46.657 [DEBUG - kudu-nio-2] (Connection.java:239) [peer master-127.3.46.60:36127(127.3.46.60:36127)] handling channelInactive
20:27:46.671 [WARN - pool-40-thread-1] (HashedWheelTimer.java:716) An exception was thrown by TimerTask.
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713)
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701)
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:787)
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501)
at java.base/java.lang.Thread.run(Thread.java:840)
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713)
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701)
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:787)
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501)
at java.base/java.lang.Thread.run(Thread.java:840)
20:27:46.672 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:46.672 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:46.673 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:46.673 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:46.673 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:46.673 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:46.673 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:46.673 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:46.674 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:46.690 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:46.691 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:46.691 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:46.691 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:46.692 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:46.692 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:46.692 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:46.692 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:46.692 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:46.711 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:46.711 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:46.712 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:46.712 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:46.712 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:46.712 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:46.712 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:46.713 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:46.713 [WARN - kudu-nio-4] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:46.731 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:46.731 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:46.732 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:46.732 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:46.732 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:46.732 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:46.732 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:46.732 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:46.732 [WARN - kudu-nio-6] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:46.751 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:46.753 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:46.753 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:46.753 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:46.753 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:46.754 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:46.754 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:46.754 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:46.754 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:46.811 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:46.811 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:46.812 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:46.812 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:46.812 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:46.812 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:46.812 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:46.812 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:46.813 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:46.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:46.840629 3706 heartbeater.cc:499] Master 127.3.46.60:36127 was elected leader, sending a full tablet report...
20:27:46.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:46.868573 4264 raft_consensus.cc:493] T 00000000000000000000000000000000 P 0db1d539946942f794b0c4ec81f8f4b4 [term 4 FOLLOWER]: Starting pre-election (detected failure of leader b1731f8308c145f686f88efd6a54b851)
20:27:46.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:46.868793 4264 raft_consensus.cc:515] T 00000000000000000000000000000000 P 0db1d539946942f794b0c4ec81f8f4b4 [term 4 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0db1d539946942f794b0c4ec81f8f4b4" member_type: VOTER last_known_addr { host: "127.3.46.62" port: 46071 } } peers { permanent_uuid: "d6c270b387b04f17b2a58bfcb0eb4f82" member_type: VOTER last_known_addr { host: "127.3.46.61" port: 39407 } } peers { permanent_uuid: "b1731f8308c145f686f88efd6a54b851" member_type: VOTER last_known_addr { host: "127.3.46.60" port: 36127 } }
20:27:46.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:46.869582 4264 leader_election.cc:290] T 00000000000000000000000000000000 P 0db1d539946942f794b0c4ec81f8f4b4 [CANDIDATE]: Term 5 pre-election: Requested pre-vote from peers d6c270b387b04f17b2a58bfcb0eb4f82 (127.3.46.61:39407), b1731f8308c145f686f88efd6a54b851 (127.3.46.60:36127)
20:27:46.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:27:46.871631 4148 leader_election.cc:336] T 00000000000000000000000000000000 P 0db1d539946942f794b0c4ec81f8f4b4 [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer b1731f8308c145f686f88efd6a54b851 (127.3.46.60:36127): Network error: Client connection negotiation failed: client connection to 127.3.46.60:36127: connect: Connection refused (error 111)
20:27:46.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:27:46.871889 4149 leader_election.cc:336] T 00000000000000000000000000000000 P 0db1d539946942f794b0c4ec81f8f4b4 [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer d6c270b387b04f17b2a58bfcb0eb4f82 (127.3.46.61:39407): Network error: Client connection negotiation failed: client connection to 127.3.46.61:39407: connect: Connection refused (error 111)
20:27:46.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:46.872001 4149 leader_election.cc:304] T 00000000000000000000000000000000 P 0db1d539946942f794b0c4ec81f8f4b4 [CANDIDATE]: Term 5 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 0db1d539946942f794b0c4ec81f8f4b4; no voters: b1731f8308c145f686f88efd6a54b851, d6c270b387b04f17b2a58bfcb0eb4f82
20:27:46.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:46.872277 4264 raft_consensus.cc:2749] T 00000000000000000000000000000000 P 0db1d539946942f794b0c4ec81f8f4b4 [term 4 FOLLOWER]: Leader pre-election lost for term 5. Reason: could not achieve majority
20:27:46.890 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:46.891 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:46.891 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:46.891 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:46.892 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:46.892 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:46.892 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:46.892 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:46.892 [WARN - kudu-nio-4] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:46.893 [DEBUG - kudu-nio-4] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] sending RPC to server master-127.3.46.60:36127, [4ms] received response from server master-127.3.46.60:36127: Network error: [peer master-127.3.46.60:36127(127.3.46.60:36127)] unexpected exception from downstream on [id: 0x8609ca1e, L:/127.0.0.1:54942 - R:/127.3.46.60:36127], [5ms] delaying RPC due to: Network error: [peer master-127.3.46.60:36127(127.3.46.60:36127)] unexpected exception from downstream on [id: 0x8609ca1e, L:/127.0.0.1:54942 - R:/127.3.46.60:36127], [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [22ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [23ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [23ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [24ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [24ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [63ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [103ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [103ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [103ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [104ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [104ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [163ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [242ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@1377780638(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$233/0x00007f8b98336cd0@33948d7c -> (continuation of Deferred@1985989261 after retry RPC after error@1559508644) -> (continuation of Deferred@1952423191 after retry RPC after error@1672937869) -> (continuation of Deferred@1807000788 after retry RPC after error@378076688) -> (continuation of Deferred@515337971 after retry RPC after error@1171763137) -> (continuation of Deferred@193671595 after retry RPC after error@1520576180) -> (continuation of Deferred@2135362533 after retry RPC after error@118889067), errback=org.apache.kudu.client.KuduTransaction$$Lambda$234/0x00007f8b98336ef8@55c2754b -> (continuation of Deferred@1985989261 after retry RPC after error@1559508644) -> (continuation of Deferred@1952423191 after retry RPC after error@1672937869) -> (continuation of Deferred@1807000788 after retry RPC after error@378076688) -> (continuation of Deferred@515337971 after retry RPC after error@1171763137) -> (continuation of Deferred@193671595 after retry RPC after error@1520576180) -> (continuation of Deferred@2135362533 after retry RPC after error@118889067)))
20:27:46.893 [DEBUG - kudu-nio-4] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] sending RPC to server master-127.3.46.60:36127, [4ms] received response from server master-127.3.46.60:36127: Network error: [peer master-127.3.46.60:36127(127.3.46.60:36127)] unexpected exception from downstream on [id: 0x8609ca1e, L:/127.0.0.1:54942 - R:/127.3.46.60:36127], [5ms] delaying RPC due to: Network error: [peer master-127.3.46.60:36127(127.3.46.60:36127)] unexpected exception from downstream on [id: 0x8609ca1e, L:/127.0.0.1:54942 - R:/127.3.46.60:36127], [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [22ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [23ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [23ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [24ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [24ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [63ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [103ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [103ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [103ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [104ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [104ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [163ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [242ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@1377780638(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$233/0x00007f8b98336cd0@33948d7c -> (continuation of Deferred@1985989261 after retry RPC after error@1559508644) -> (continuation of Deferred@1952423191 after retry RPC after error@1672937869) -> (continuation of Deferred@1807000788 after retry RPC after error@378076688) -> (continuation of Deferred@515337971 after retry RPC after error@1171763137) -> (continuation of Deferred@193671595 after retry RPC after error@1520576180) -> (continuation of Deferred@2135362533 after retry RPC after error@118889067), errback=org.apache.kudu.client.KuduTransaction$$Lambda$234/0x00007f8b98336ef8@55c2754b -> (continuation of Deferred@1985989261 after retry RPC after error@1559508644) -> (continuation of Deferred@1952423191 after retry RPC after error@1672937869) -> (continuation of Deferred@1807000788 after retry RPC after error@378076688) -> (continuation of Deferred@515337971 after retry RPC after error@1171763137) -> (continuation of Deferred@193671595 after retry RPC after error@1520576180) -> (continuation of Deferred@2135362533 after retry RPC after error@118889067)))
20:27:46.911 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:46.911 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:46.912 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:46.912 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:46.912 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:46.912 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:46.912 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:46.912 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:46.912 [WARN - kudu-nio-6] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:46.930 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:46.931 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:46.931 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:46.932 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:46.932 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:46.932 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:46.932 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:46.932 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:46.932 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:46.951 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:46.952 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:46.953 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:46.953 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:46.953 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:46.954 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:46.954 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:46.954 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:46.954 [WARN - kudu-nio-2] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:46.971 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:46.971 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:46.972 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:46.972 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:46.972 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:46.972 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:46.972 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:46.972 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:46.972 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:46.991 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:46.992 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:46.992 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:46.992 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:46.992 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:46.992 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:46.993 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:46.993 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:46.993 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:47.011 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:47.011 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:47.012 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:47.012 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:47.012 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:47.012 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:47.012 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:47.012 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:47.013 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:47.071 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:47.071 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:47.072 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:47.072 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:47.072 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:47.072 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:47.072 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:47.072 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:47.072 [WARN - kudu-nio-2] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:47.151 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:47.151 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:47.152 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:47.152 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:47.153 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:47.153 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:47.153 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:47.153 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:47.153 [WARN - kudu-nio-4] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:47.154 [DEBUG - kudu-nio-4] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=243), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [43ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [43ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [44ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [44ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [82ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [83ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [83ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [83ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [100ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [243ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [243ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [243ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@1159240145(state=PENDING, result=null, callback=(continuation of Deferred@381419584 after retry RPC after error@1722385712) -> (continuation of Deferred@585350570 after retry RPC after error@599031350) -> (continuation of Deferred@1434244096 after retry RPC after error@2144750490) -> (continuation of Deferred@1088927527 after retry RPC after error@1084368067) -> (continuation of Deferred@1934439427 after retry RPC after error@1049246571) -> (continuation of Deferred@1126825234 after retry RPC after error@310136200) -> (continuation of Deferred@134979304 after retry RPC after error@1739959777), errback=(continuation of Deferred@381419584 after retry RPC after error@1722385712) -> (continuation of Deferred@585350570 after retry RPC after error@599031350) -> (continuation of Deferred@1434244096 after retry RPC after error@2144750490) -> (continuation of Deferred@1088927527 after retry RPC after error@1084368067) -> (continuation of Deferred@1934439427 after retry RPC after error@1049246571) -> (continuation of Deferred@1126825234 after retry RPC after error@310136200) -> (continuation of Deferred@134979304 after retry RPC after error@1739959777)))
20:27:47.154 [DEBUG - kudu-nio-4] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=243), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [43ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [43ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [44ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [44ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [82ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [83ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [83ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [83ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [100ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [243ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [243ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [243ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@1159240145(state=PENDING, result=null, callback=(continuation of Deferred@381419584 after retry RPC after error@1722385712) -> (continuation of Deferred@585350570 after retry RPC after error@599031350) -> (continuation of Deferred@1434244096 after retry RPC after error@2144750490) -> (continuation of Deferred@1088927527 after retry RPC after error@1084368067) -> (continuation of Deferred@1934439427 after retry RPC after error@1049246571) -> (continuation of Deferred@1126825234 after retry RPC after error@310136200) -> (continuation of Deferred@134979304 after retry RPC after error@1739959777), errback=(continuation of Deferred@381419584 after retry RPC after error@1722385712) -> (continuation of Deferred@585350570 after retry RPC after error@599031350) -> (continuation of Deferred@1434244096 after retry RPC after error@2144750490) -> (continuation of Deferred@1088927527 after retry RPC after error@1084368067) -> (continuation of Deferred@1934439427 after retry RPC after error@1049246571) -> (continuation of Deferred@1126825234 after retry RPC after error@310136200) -> (continuation of Deferred@134979304 after retry RPC after error@1739959777)))
20:27:47.171 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:47.172 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:47.172 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:47.172 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:47.173 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:47.173 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:47.173 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:47.173 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:47.173 [WARN - kudu-nio-6] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:47.175 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:47.175 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:47.176 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:47.176 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:47.176 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:47.176 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:47.176 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:47.176 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:47.177 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:47.191 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:47.191 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:47.192 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:47.192 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:47.192 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:47.192 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:47.192 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:47.192 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:47.192 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:47.192 [WARN - kudu-nio-2] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:47.192 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:47.193 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:47.210 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:47.211 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:47.211 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:47.211 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:47.211 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:47.211 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:47.211 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:47.211 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:47.211 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:47.211 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:47.212 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:47.213 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:47.231 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:47.231 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:47.232 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:47.232 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:47.232 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:47.232 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:47.232 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:47.232 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:47.232 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:47.232 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:47.233 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:47.233 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:47.250 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:47.251 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:47.251 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:47.251 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:47.251 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:47.252 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:47.252 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:47.252 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:47.252 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:47.252 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:47.252 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:47.252 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:47.252 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:47.252 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:47.253 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:47.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:47.269912 3848 heartbeater.cc:499] Master 127.3.46.60:36127 was elected leader, sending a full tablet report...
20:27:47.271 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:47.271 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:47.271 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:47.271 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:47.272 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:47.272 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:47.272 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:47.272 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:47.273 [WARN - kudu-nio-3] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:47.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:47.281678 3874 txn_status_manager.cc:1391] automatically aborted stale txn (ID 0) past 1.137s from last keepalive heartbeat (effective timeout is 1.000s)
20:27:47.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:27:47.290896 3893 master_proxy_rpc.cc:203] Re-attempting LookupRpcById request to leader Master (127.3.46.62:46071)
20:27:47.291 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:47.292 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:47.292 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:47.292 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:47.292 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:47.293 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:47.292 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:47.293 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:47.293 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:47.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:47.309942 4264 raft_consensus.cc:493] T 00000000000000000000000000000000 P 0db1d539946942f794b0c4ec81f8f4b4 [term 4 FOLLOWER]: Starting pre-election (detected failure of leader b1731f8308c145f686f88efd6a54b851)
20:27:47.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:47.310144 4264 raft_consensus.cc:515] T 00000000000000000000000000000000 P 0db1d539946942f794b0c4ec81f8f4b4 [term 4 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0db1d539946942f794b0c4ec81f8f4b4" member_type: VOTER last_known_addr { host: "127.3.46.62" port: 46071 } } peers { permanent_uuid: "d6c270b387b04f17b2a58bfcb0eb4f82" member_type: VOTER last_known_addr { host: "127.3.46.61" port: 39407 } } peers { permanent_uuid: "b1731f8308c145f686f88efd6a54b851" member_type: VOTER last_known_addr { host: "127.3.46.60" port: 36127 } }
20:27:47.310 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:47.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:47.310760 4264 leader_election.cc:290] T 00000000000000000000000000000000 P 0db1d539946942f794b0c4ec81f8f4b4 [CANDIDATE]: Term 5 pre-election: Requested pre-vote from peers d6c270b387b04f17b2a58bfcb0eb4f82 (127.3.46.61:39407), b1731f8308c145f686f88efd6a54b851 (127.3.46.60:36127)
20:27:47.311 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:47.311 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:47.311 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:47.312 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:47.312 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:47.312 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:47.312 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:47.313 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:47.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:27:47.313365 4148 leader_election.cc:336] T 00000000000000000000000000000000 P 0db1d539946942f794b0c4ec81f8f4b4 [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer b1731f8308c145f686f88efd6a54b851 (127.3.46.60:36127): Network error: Client connection negotiation failed: client connection to 127.3.46.60:36127: connect: Connection refused (error 111)
20:27:47.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:27:47.313599 4149 leader_election.cc:336] T 00000000000000000000000000000000 P 0db1d539946942f794b0c4ec81f8f4b4 [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer d6c270b387b04f17b2a58bfcb0eb4f82 (127.3.46.61:39407): Network error: Client connection negotiation failed: client connection to 127.3.46.61:39407: connect: Connection refused (error 111)
20:27:47.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:47.313748 4149 leader_election.cc:304] T 00000000000000000000000000000000 P 0db1d539946942f794b0c4ec81f8f4b4 [CANDIDATE]: Term 5 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 0db1d539946942f794b0c4ec81f8f4b4; no voters: b1731f8308c145f686f88efd6a54b851, d6c270b387b04f17b2a58bfcb0eb4f82
20:27:47.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:47.314101 4264 raft_consensus.cc:2749] T 00000000000000000000000000000000 P 0db1d539946942f794b0c4ec81f8f4b4 [term 4 FOLLOWER]: Leader pre-election lost for term 5. Reason: could not achieve majority
20:27:47.351 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:47.351 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:47.352 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:47.352 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:47.352 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:47.352 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:47.352 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:47.352 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:47.352 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:47.390 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:47.391 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:47.391 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:47.391 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:47.392 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:47.392 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:47.392 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:47.392 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:47.392 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:47.411 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:47.412 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:47.412 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:47.412 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:47.412 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:47.412 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:47.412 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:47.412 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:47.413 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:47.413 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [40ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [80ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [99ms] refreshing cache from master, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [139ms] refreshing cache from master, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [140ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [239ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@2002877253(state=PENDING, result=null, callback=(continuation of Deferred@451640840 after retry RPC after error@2008649451) -> (continuation of Deferred@74864964 after retry RPC after error@2024907228) -> (continuation of Deferred@971742204 after retry RPC after error@43947260) -> (continuation of Deferred@873746834 after retry RPC after error@1105111437) -> (continuation of Deferred@605023793 after retry RPC after error@1388361528) -> (continuation of Deferred@1154409523 after retry RPC after error@1894297574) -> (continuation of Deferred@726514092 after retry RPC after error@2086904458), errback=(continuation of Deferred@451640840 after retry RPC after error@2008649451) -> (continuation of Deferred@74864964 after retry RPC after error@2024907228) -> (continuation of Deferred@971742204 after retry RPC after error@43947260) -> (continuation of Deferred@873746834 after retry RPC after error@1105111437) -> (continuation of Deferred@605023793 after retry RPC after error@1388361528) -> (continuation of Deferred@1154409523 after retry RPC after error@1894297574) -> (continuation of Deferred@726514092 after retry RPC after error@2086904458)))
20:27:47.414 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [40ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [80ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [99ms] refreshing cache from master, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [139ms] refreshing cache from master, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [140ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [239ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@2002877253(state=PENDING, result=null, callback=(continuation of Deferred@451640840 after retry RPC after error@2008649451) -> (continuation of Deferred@74864964 after retry RPC after error@2024907228) -> (continuation of Deferred@971742204 after retry RPC after error@43947260) -> (continuation of Deferred@873746834 after retry RPC after error@1105111437) -> (continuation of Deferred@605023793 after retry RPC after error@1388361528) -> (continuation of Deferred@1154409523 after retry RPC after error@1894297574) -> (continuation of Deferred@726514092 after retry RPC after error@2086904458), errback=(continuation of Deferred@451640840 after retry RPC after error@2008649451) -> (continuation of Deferred@74864964 after retry RPC after error@2024907228) -> (continuation of Deferred@971742204 after retry RPC after error@43947260) -> (continuation of Deferred@873746834 after retry RPC after error@1105111437) -> (continuation of Deferred@605023793 after retry RPC after error@1388361528) -> (continuation of Deferred@1154409523 after retry RPC after error@1894297574) -> (continuation of Deferred@726514092 after retry RPC after error@2086904458)))
20:27:47.430 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:47.431 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:47.432 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:47.432 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:47.432 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:47.432 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:47.432 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:47.432 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:47.432 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:47.451 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:47.451 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:47.452 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:47.452 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:47.452 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:47.452 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:47.452 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:47.452 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:47.452 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:47.471 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:47.472 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:47.472 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:47.472 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:47.472 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:47.472 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:47.472 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:47.473 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:47.473 [WARN - kudu-nio-3] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:47.490 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:47.491 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:47.491 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:47.491 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:47.492 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:47.492 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:47.492 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:47.492 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:47.493 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:47.511 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:47.511 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:47.512 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:47.512 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:47.512 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:47.512 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:47.512 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:47.512 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:47.513 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:47.531 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:47.532 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:47.532 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:47.532 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:47.532 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:47.533 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:47.533 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:47.533 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:47.533 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:47.571 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:47.571 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:47.572 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:47.572 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:47.572 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:47.572 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:47.572 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:47.572 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:47.572 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:47.573 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=142), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [20ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [42ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [43ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [63ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [83ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [102ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [103ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [103ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@1807141780(state=PENDING, result=null, callback=(continuation of Deferred@2026283682 after retry RPC after error@531974330) -> (continuation of Deferred@764552485 after retry RPC after error@1881514344) -> (continuation of Deferred@1035718268 after retry RPC after error@150905321) -> (continuation of Deferred@888735454 after retry RPC after error@1770704840) -> (continuation of Deferred@178332843 after retry RPC after error@1999449599) -> (continuation of Deferred@186518066 after retry RPC after error@886403094), errback=(continuation of Deferred@2026283682 after retry RPC after error@531974330) -> (continuation of Deferred@764552485 after retry RPC after error@1881514344) -> (continuation of Deferred@1035718268 after retry RPC after error@150905321) -> (continuation of Deferred@888735454 after retry RPC after error@1770704840) -> (continuation of Deferred@178332843 after retry RPC after error@1999449599) -> (continuation of Deferred@186518066 after retry RPC after error@886403094)))
20:27:47.573 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=142), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [20ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [42ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [43ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [63ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [83ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [102ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [103ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [103ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@1807141780(state=PENDING, result=null, callback=(continuation of Deferred@2026283682 after retry RPC after error@531974330) -> (continuation of Deferred@764552485 after retry RPC after error@1881514344) -> (continuation of Deferred@1035718268 after retry RPC after error@150905321) -> (continuation of Deferred@888735454 after retry RPC after error@1770704840) -> (continuation of Deferred@178332843 after retry RPC after error@1999449599) -> (continuation of Deferred@186518066 after retry RPC after error@886403094), errback=(continuation of Deferred@2026283682 after retry RPC after error@531974330) -> (continuation of Deferred@764552485 after retry RPC after error@1881514344) -> (continuation of Deferred@1035718268 after retry RPC after error@150905321) -> (continuation of Deferred@888735454 after retry RPC after error@1770704840) -> (continuation of Deferred@178332843 after retry RPC after error@1999449599) -> (continuation of Deferred@186518066 after retry RPC after error@886403094)))
20:27:47.591 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:47.592 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:47.592 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:47.593 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:47.593 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:47.593 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:47.593 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:47.593 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:47.593 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:47.611 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:47.612 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:47.612 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:47.612 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:47.612 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:47.612 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:47.612 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:47.612 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:47.613 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:47.613 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:47.613 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:47.614 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:47.631 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:47.631 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:47.632 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:47.632 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:47.632 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:47.632 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:47.632 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:47.632 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:47.632 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:47.650 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:47.651 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:47.651 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:47.651 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:47.651 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:47.652 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:47.652 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:47.652 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:47.652 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:47.671 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:47.671 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:47.672 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:47.672 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:47.672 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:47.672 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:47.672 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:47.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:47.672734 4264 raft_consensus.cc:493] T 00000000000000000000000000000000 P 0db1d539946942f794b0c4ec81f8f4b4 [term 4 FOLLOWER]: Starting pre-election (detected failure of leader b1731f8308c145f686f88efd6a54b851)
20:27:47.673 [INFO - pool-40-thread-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:47.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:47.672940 4264 raft_consensus.cc:515] T 00000000000000000000000000000000 P 0db1d539946942f794b0c4ec81f8f4b4 [term 4 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0db1d539946942f794b0c4ec81f8f4b4" member_type: VOTER last_known_addr { host: "127.3.46.62" port: 46071 } } peers { permanent_uuid: "d6c270b387b04f17b2a58bfcb0eb4f82" member_type: VOTER last_known_addr { host: "127.3.46.61" port: 39407 } } peers { permanent_uuid: "b1731f8308c145f686f88efd6a54b851" member_type: VOTER last_known_addr { host: "127.3.46.60" port: 36127 } }
20:27:47.673 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:47.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:47.673866 4264 leader_election.cc:290] T 00000000000000000000000000000000 P 0db1d539946942f794b0c4ec81f8f4b4 [CANDIDATE]: Term 5 pre-election: Requested pre-vote from peers d6c270b387b04f17b2a58bfcb0eb4f82 (127.3.46.61:39407), b1731f8308c145f686f88efd6a54b851 (127.3.46.60:36127)
20:27:47.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:27:47.675246 4149 leader_election.cc:336] T 00000000000000000000000000000000 P 0db1d539946942f794b0c4ec81f8f4b4 [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer d6c270b387b04f17b2a58bfcb0eb4f82 (127.3.46.61:39407): Network error: Client connection negotiation failed: client connection to 127.3.46.61:39407: connect: Connection refused (error 111)
20:27:47.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:27:47.675479 4148 leader_election.cc:336] T 00000000000000000000000000000000 P 0db1d539946942f794b0c4ec81f8f4b4 [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer b1731f8308c145f686f88efd6a54b851 (127.3.46.60:36127): Network error: Client connection negotiation failed: client connection to 127.3.46.60:36127: connect: Connection refused (error 111)
20:27:47.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:47.675796 4148 leader_election.cc:304] T 00000000000000000000000000000000 P 0db1d539946942f794b0c4ec81f8f4b4 [CANDIDATE]: Term 5 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 0db1d539946942f794b0c4ec81f8f4b4; no voters: b1731f8308c145f686f88efd6a54b851, d6c270b387b04f17b2a58bfcb0eb4f82
20:27:47.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:47.676102 4264 raft_consensus.cc:2749] T 00000000000000000000000000000000 P 0db1d539946942f794b0c4ec81f8f4b4 [term 4 FOLLOWER]: Leader pre-election lost for term 5. Reason: could not achieve majority
20:27:47.691 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:47.692 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:47.692 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:47.692 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:47.692 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:47.692 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:47.692 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:47.692 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:47.693 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:47.751 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:47.751 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:47.752 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:47.752 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:47.752 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:47.752 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:47.752 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:47.753 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:47.753 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:47.770 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:47.771 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:47.771 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:47.771 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:47.772 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:47.772 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:47.772 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:47.772 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:47.772 [WARN - kudu-nio-3] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:47.772 [DEBUG - kudu-nio-3] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=181), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [60ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [159ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [179ms] refreshing cache from master, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [181ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@1743135405(state=PENDING, result=null, callback=(continuation of Deferred@1676919761 after retry RPC after error@1040160008) -> (continuation of Deferred@45168840 after retry RPC after error@1419022713) -> (continuation of Deferred@696532337 after retry RPC after error@1746053788) -> (continuation of Deferred@1668196398 after retry RPC after error@327279203) -> (continuation of Deferred@459680471 after retry RPC after error@1725769069) -> (continuation of Deferred@1894043200 after retry RPC after error@1827322053) -> (continuation of Deferred@1190139677 after retry RPC after error@1225368803), errback=(continuation of Deferred@1676919761 after retry RPC after error@1040160008) -> (continuation of Deferred@45168840 after retry RPC after error@1419022713) -> (continuation of Deferred@696532337 after retry RPC after error@1746053788) -> (continuation of Deferred@1668196398 after retry RPC after error@327279203) -> (continuation of Deferred@459680471 after retry RPC after error@1725769069) -> (continuation of Deferred@1894043200 after retry RPC after error@1827322053) -> (continuation of Deferred@1190139677 after retry RPC after error@1225368803)))
20:27:47.773 [DEBUG - kudu-nio-3] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=181), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [60ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [159ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [179ms] refreshing cache from master, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [181ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@1743135405(state=PENDING, result=null, callback=(continuation of Deferred@1676919761 after retry RPC after error@1040160008) -> (continuation of Deferred@45168840 after retry RPC after error@1419022713) -> (continuation of Deferred@696532337 after retry RPC after error@1746053788) -> (continuation of Deferred@1668196398 after retry RPC after error@327279203) -> (continuation of Deferred@459680471 after retry RPC after error@1725769069) -> (continuation of Deferred@1894043200 after retry RPC after error@1827322053) -> (continuation of Deferred@1190139677 after retry RPC after error@1225368803), errback=(continuation of Deferred@1676919761 after retry RPC after error@1040160008) -> (continuation of Deferred@45168840 after retry RPC after error@1419022713) -> (continuation of Deferred@696532337 after retry RPC after error@1746053788) -> (continuation of Deferred@1668196398 after retry RPC after error@327279203) -> (continuation of Deferred@459680471 after retry RPC after error@1725769069) -> (continuation of Deferred@1894043200 after retry RPC after error@1827322053) -> (continuation of Deferred@1190139677 after retry RPC after error@1225368803)))
20:27:47.791 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:47.791 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:47.792 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:47.792 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:47.792 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:47.792 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:47.792 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:47.792 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:47.792 [WARN - kudu-nio-5] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:47.810 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:47.811 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:47.811 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:47.811 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:47.812 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:47.812 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:47.812 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:47.812 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:47.812 [WARN - kudu-nio-7] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:47.831 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:47.832 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:47.832 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:47.832 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:47.832 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:47.832 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:47.832 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:47.832 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:47.833 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:47.850 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:47.851 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:47.851 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:47.851 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:47.851 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:47.851 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:47.852 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:47.852 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:47.852 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:47.871 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:47.871 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:47.871 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:47.872 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:47.872 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:47.872 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:47.872 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:47.872 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:47.872 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:47.911 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:47.912 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:47.912 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:47.912 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:47.912 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:47.912 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:47.912 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:47.913 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:47.913 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:47.971 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:47.972 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:47.972 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:47.972 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:47.972 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:47.973 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:47.973 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:47.973 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:47.973 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:47.990 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:47.991 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:47.991 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:47.991 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:47.992 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:47.992 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:47.992 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:47.992 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:47.993 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:47.994 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [3ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [43ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [122ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [123ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [181ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [183ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [183ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [203ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [203ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@266112183(state=PENDING, result=null, callback=(continuation of Deferred@2093939705 after retry RPC after error@1072936664) -> (continuation of Deferred@1848801116 after retry RPC after error@90236697) -> (continuation of Deferred@251315583 after retry RPC after error@740157198) -> (continuation of Deferred@1374223686 after retry RPC after error@1404207181) -> (continuation of Deferred@188442233 after retry RPC after error@1222497671) -> (continuation of Deferred@1590231831 after retry RPC after error@431898731) -> (continuation of Deferred@1231480203 after retry RPC after error@1865507021), errback=(continuation of Deferred@2093939705 after retry RPC after error@1072936664) -> (continuation of Deferred@1848801116 after retry RPC after error@90236697) -> (continuation of Deferred@251315583 after retry RPC after error@740157198) -> (continuation of Deferred@1374223686 after retry RPC after error@1404207181) -> (continuation of Deferred@188442233 after retry RPC after error@1222497671) -> (continuation of Deferred@1590231831 after retry RPC after error@431898731) -> (continuation of Deferred@1231480203 after retry RPC after error@1865507021)))
20:27:47.994 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [3ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [43ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [122ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [123ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [181ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [183ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [183ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [203ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [203ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@266112183(state=PENDING, result=null, callback=(continuation of Deferred@2093939705 after retry RPC after error@1072936664) -> (continuation of Deferred@1848801116 after retry RPC after error@90236697) -> (continuation of Deferred@251315583 after retry RPC after error@740157198) -> (continuation of Deferred@1374223686 after retry RPC after error@1404207181) -> (continuation of Deferred@188442233 after retry RPC after error@1222497671) -> (continuation of Deferred@1590231831 after retry RPC after error@431898731) -> (continuation of Deferred@1231480203 after retry RPC after error@1865507021), errback=(continuation of Deferred@2093939705 after retry RPC after error@1072936664) -> (continuation of Deferred@1848801116 after retry RPC after error@90236697) -> (continuation of Deferred@251315583 after retry RPC after error@740157198) -> (continuation of Deferred@1374223686 after retry RPC after error@1404207181) -> (continuation of Deferred@188442233 after retry RPC after error@1222497671) -> (continuation of Deferred@1590231831 after retry RPC after error@431898731) -> (continuation of Deferred@1231480203 after retry RPC after error@1865507021)))
20:27:48.011 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:48.012 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:48.012 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:48.012 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:48.012 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:48.012 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:48.012 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:48.013 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:48.013 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:48.030 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:48.031 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:48.031 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:48.031 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:48.031 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:48.032 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:48.032 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:48.032 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:48.032 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:48.051 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:48.052 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:48.052 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:48.053 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:48.053 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:48.053 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:48.053 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:48.053 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:48.053 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:48.054 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:48.054 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:48.054 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:48.071 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:48.071 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:48.072 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:48.072 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:48.072 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:48.072 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:48.073 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:48.073 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:48.073 [WARN - kudu-nio-3] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:48.091 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:48.092 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:48.092 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:48.092 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:48.092 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:48.092 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:48.092 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:48.092 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:48.093 [WARN - kudu-nio-4] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:48.130 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:48.131 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:48.132 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:48.132 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:48.132 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:48.132 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:48.132 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:48.132 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:48.132 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:48.190 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:48.191 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:48.191 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:48.191 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:48.191 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:48.191 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:48.191 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:48.192 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:48.192 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:48.192 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=181), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [3ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] refreshing cache from master, [42ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [42ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [42ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [43ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [44ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [44ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [44ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [60ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [63ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [63ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [63ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@402814627(state=PENDING, result=null, callback=(continuation of Deferred@731998158 after retry RPC after error@1324495363) -> (continuation of Deferred@1904098600 after retry RPC after error@2051209930) -> (continuation of Deferred@1518173531 after retry RPC after error@96480829) -> (continuation of Deferred@89638096 after retry RPC after error@1756297754) -> (continuation of Deferred@1493747185 after retry RPC after error@1572714795) -> (continuation of Deferred@1719109384 after retry RPC after error@2031538166), errback=(continuation of Deferred@731998158 after retry RPC after error@1324495363) -> (continuation of Deferred@1904098600 after retry RPC after error@2051209930) -> (continuation of Deferred@1518173531 after retry RPC after error@96480829) -> (continuation of Deferred@89638096 after retry RPC after error@1756297754) -> (continuation of Deferred@1493747185 after retry RPC after error@1572714795) -> (continuation of Deferred@1719109384 after retry RPC after error@2031538166)))
20:27:48.193 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=181), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [3ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] refreshing cache from master, [42ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [42ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [42ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [43ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [44ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [44ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [44ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [60ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [63ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [63ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [63ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@402814627(state=PENDING, result=null, callback=(continuation of Deferred@731998158 after retry RPC after error@1324495363) -> (continuation of Deferred@1904098600 after retry RPC after error@2051209930) -> (continuation of Deferred@1518173531 after retry RPC after error@96480829) -> (continuation of Deferred@89638096 after retry RPC after error@1756297754) -> (continuation of Deferred@1493747185 after retry RPC after error@1572714795) -> (continuation of Deferred@1719109384 after retry RPC after error@2031538166), errback=(continuation of Deferred@731998158 after retry RPC after error@1324495363) -> (continuation of Deferred@1904098600 after retry RPC after error@2051209930) -> (continuation of Deferred@1518173531 after retry RPC after error@96480829) -> (continuation of Deferred@89638096 after retry RPC after error@1756297754) -> (continuation of Deferred@1493747185 after retry RPC after error@1572714795) -> (continuation of Deferred@1719109384 after retry RPC after error@2031538166)))
20:27:48.211 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:48.211 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:48.212 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:48.212 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:48.212 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:48.214 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:48.214 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:48.214 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:48.214 [WARN - kudu-nio-3] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:48.231 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:48.232 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:48.232 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:48.232 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:48.232 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:48.232 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:48.232 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:48.232 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:48.233 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:48.250 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:48.251 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:48.251 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:48.251 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:48.252 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:48.252 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:48.252 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:48.252 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:48.252 [WARN - kudu-nio-7] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:48.271 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:48.271 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:48.272 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:48.272 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:48.272 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:48.272 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:48.272 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:48.272 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:48.272 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:48.291 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:48.292 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:48.292 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:48.292 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:48.292 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:48.292 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:48.292 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:48.292 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:48.293 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:48.310 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:48.311 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:48.311 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:48.311 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:48.312 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:48.312 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:48.312 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:48.312 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:48.312 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:48.351 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:48.352 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:48.352 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:48.352 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:48.352 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:48.352 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:48.352 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:48.352 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:48.353 [WARN - kudu-nio-7] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:48.353 [DEBUG - kudu-nio-7] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=142), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [3ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [4ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [4ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [22ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [23ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [83ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [141ms] refreshing cache from master, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [142ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [143ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@1868210934(state=PENDING, result=null, callback=(continuation of Deferred@1756418517 after retry RPC after error@2017716813) -> (continuation of Deferred@838832130 after retry RPC after error@185129433) -> (continuation of Deferred@1700306708 after retry RPC after error@36569200) -> (continuation of Deferred@1327300535 after retry RPC after error@43458972) -> (continuation of Deferred@597081829 after retry RPC after error@1372129127) -> (continuation of Deferred@2038479790 after retry RPC after error@1117843139), errback=(continuation of Deferred@1756418517 after retry RPC after error@2017716813) -> (continuation of Deferred@838832130 after retry RPC after error@185129433) -> (continuation of Deferred@1700306708 after retry RPC after error@36569200) -> (continuation of Deferred@1327300535 after retry RPC after error@43458972) -> (continuation of Deferred@597081829 after retry RPC after error@1372129127) -> (continuation of Deferred@2038479790 after retry RPC after error@1117843139)))
20:27:48.353 [DEBUG - kudu-nio-7] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=142), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [3ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [4ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [4ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [22ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [23ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [83ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [141ms] refreshing cache from master, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [142ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [143ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@1868210934(state=PENDING, result=null, callback=(continuation of Deferred@1756418517 after retry RPC after error@2017716813) -> (continuation of Deferred@838832130 after retry RPC after error@185129433) -> (continuation of Deferred@1700306708 after retry RPC after error@36569200) -> (continuation of Deferred@1327300535 after retry RPC after error@43458972) -> (continuation of Deferred@597081829 after retry RPC after error@1372129127) -> (continuation of Deferred@2038479790 after retry RPC after error@1117843139), errback=(continuation of Deferred@1756418517 after retry RPC after error@2017716813) -> (continuation of Deferred@838832130 after retry RPC after error@185129433) -> (continuation of Deferred@1700306708 after retry RPC after error@36569200) -> (continuation of Deferred@1327300535 after retry RPC after error@43458972) -> (continuation of Deferred@597081829 after retry RPC after error@1372129127) -> (continuation of Deferred@2038479790 after retry RPC after error@1117843139)))
20:27:48.370 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:48.371 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:48.371 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:48.371 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:48.372 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:48.372 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:48.372 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:48.372 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:48.372 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:48.391 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:48.391 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:48.392 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:48.392 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:48.392 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:48.392 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:48.392 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:48.392 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:48.393 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:48.410 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:48.411 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:48.411 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:48.411 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:48.411 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:48.411 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:48.411 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:48.412 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:48.412 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:48.431 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:48.431 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:48.431 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:48.432 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:48.432 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:48.432 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:48.432 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:48.432 [INFO - pool-40-thread-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:48.432 [WARN - pool-40-thread-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:48.451 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:48.451 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:48.452 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:48.452 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:48.452 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:48.452 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:48.452 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:48.452 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:48.452 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:48.470 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:48.471 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:48.471 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:48.471 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:48.471 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:48.472 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:48.472 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:48.472 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:48.472 [WARN - kudu-nio-3] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:48.491 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:48.496 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:48.496 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:48.496 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:48.496 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:48.496 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:48.497 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:48.497 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:48.497 [WARN - kudu-nio-5] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:48.551 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:48.552 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:48.552 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:48.552 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:48.552 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:48.552 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:48.552 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:48.552 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:48.553 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:48.571 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:48.571 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:48.571 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:48.571 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:48.571 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:48.572 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:48.572 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:48.572 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:48.573 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:48.573 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [126ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [126ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [126ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [127ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [127ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [181ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [182ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [203ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@2025523418(state=PENDING, result=null, callback=(continuation of Deferred@502905268 after retry RPC after error@603513022) -> (continuation of Deferred@1555192180 after retry RPC after error@2087679968) -> (continuation of Deferred@738940352 after retry RPC after error@1217576431) -> (continuation of Deferred@103101552 after retry RPC after error@32670499) -> (continuation of Deferred@1998676872 after retry RPC after error@713147718) -> (continuation of Deferred@94958290 after retry RPC after error@274109151) -> (continuation of Deferred@1821875838 after retry RPC after error@1209584991), errback=(continuation of Deferred@502905268 after retry RPC after error@603513022) -> (continuation of Deferred@1555192180 after retry RPC after error@2087679968) -> (continuation of Deferred@738940352 after retry RPC after error@1217576431) -> (continuation of Deferred@103101552 after retry RPC after error@32670499) -> (continuation of Deferred@1998676872 after retry RPC after error@713147718) -> (continuation of Deferred@94958290 after retry RPC after error@274109151) -> (continuation of Deferred@1821875838 after retry RPC after error@1209584991)))
20:27:48.573 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [126ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [126ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [126ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [127ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [127ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [181ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [182ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [203ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@2025523418(state=PENDING, result=null, callback=(continuation of Deferred@502905268 after retry RPC after error@603513022) -> (continuation of Deferred@1555192180 after retry RPC after error@2087679968) -> (continuation of Deferred@738940352 after retry RPC after error@1217576431) -> (continuation of Deferred@103101552 after retry RPC after error@32670499) -> (continuation of Deferred@1998676872 after retry RPC after error@713147718) -> (continuation of Deferred@94958290 after retry RPC after error@274109151) -> (continuation of Deferred@1821875838 after retry RPC after error@1209584991), errback=(continuation of Deferred@502905268 after retry RPC after error@603513022) -> (continuation of Deferred@1555192180 after retry RPC after error@2087679968) -> (continuation of Deferred@738940352 after retry RPC after error@1217576431) -> (continuation of Deferred@103101552 after retry RPC after error@32670499) -> (continuation of Deferred@1998676872 after retry RPC after error@713147718) -> (continuation of Deferred@94958290 after retry RPC after error@274109151) -> (continuation of Deferred@1821875838 after retry RPC after error@1209584991)))
20:27:48.591 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:48.591 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:48.592 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:48.592 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:48.592 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:48.592 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:48.592 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:48.592 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:48.592 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:48.611 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:48.612 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:48.612 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:48.612 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:48.612 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:48.612 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:48.612 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:48.612 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:48.613 [WARN - kudu-nio-5] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:48.630 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:48.631 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:48.631 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:48.631 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:48.631 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:48.631 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:48.631 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:48.632 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:48.632 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:48.651 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:48.651 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:48.651 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:48.652 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:48.652 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:48.652 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:48.652 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:48.652 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:48.652 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:48.671 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:48.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:48.671161 4288 raft_consensus.cc:493] T 00000000000000000000000000000000 P 0db1d539946942f794b0c4ec81f8f4b4 [term 4 FOLLOWER]: Starting pre-election (detected failure of leader b1731f8308c145f686f88efd6a54b851)
20:27:48.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:48.671609 4288 raft_consensus.cc:515] T 00000000000000000000000000000000 P 0db1d539946942f794b0c4ec81f8f4b4 [term 4 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0db1d539946942f794b0c4ec81f8f4b4" member_type: VOTER last_known_addr { host: "127.3.46.62" port: 46071 } } peers { permanent_uuid: "d6c270b387b04f17b2a58bfcb0eb4f82" member_type: VOTER last_known_addr { host: "127.3.46.61" port: 39407 } } peers { permanent_uuid: "b1731f8308c145f686f88efd6a54b851" member_type: VOTER last_known_addr { host: "127.3.46.60" port: 36127 } }
20:27:48.672 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:48.672 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:48.672 [INFO - pool-40-thread-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:48.673 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:48.673 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:48.674 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:48.674 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:48.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:48.674578 4288 leader_election.cc:290] T 00000000000000000000000000000000 P 0db1d539946942f794b0c4ec81f8f4b4 [CANDIDATE]: Term 5 pre-election: Requested pre-vote from peers d6c270b387b04f17b2a58bfcb0eb4f82 (127.3.46.61:39407), b1731f8308c145f686f88efd6a54b851 (127.3.46.60:36127)
20:27:48.674 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:48.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:27:48.676000 4148 leader_election.cc:336] T 00000000000000000000000000000000 P 0db1d539946942f794b0c4ec81f8f4b4 [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer b1731f8308c145f686f88efd6a54b851 (127.3.46.60:36127): Network error: Client connection negotiation failed: client connection to 127.3.46.60:36127: connect: Connection refused (error 111)
20:27:48.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:27:48.676195 4149 leader_election.cc:336] T 00000000000000000000000000000000 P 0db1d539946942f794b0c4ec81f8f4b4 [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer d6c270b387b04f17b2a58bfcb0eb4f82 (127.3.46.61:39407): Network error: Client connection negotiation failed: client connection to 127.3.46.61:39407: connect: Connection refused (error 111)
20:27:48.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:48.676352 4149 leader_election.cc:304] T 00000000000000000000000000000000 P 0db1d539946942f794b0c4ec81f8f4b4 [CANDIDATE]: Term 5 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 0db1d539946942f794b0c4ec81f8f4b4; no voters: b1731f8308c145f686f88efd6a54b851, d6c270b387b04f17b2a58bfcb0eb4f82
20:27:48.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:48.676640 4288 raft_consensus.cc:2749] T 00000000000000000000000000000000 P 0db1d539946942f794b0c4ec81f8f4b4 [term 4 FOLLOWER]: Leader pre-election lost for term 5. Reason: could not achieve majority
20:27:48.691 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:48.691 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:48.692 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:48.692 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:48.692 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:48.692 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:48.692 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:48.692 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:48.693 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:48.731 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:48.731 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:48.732 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:48.732 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:48.732 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:48.732 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:48.732 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:48.732 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:48.733 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:48.811 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:48.811 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:48.811 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:48.811 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:48.812 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:48.812 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:48.812 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:48.812 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:48.812 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:48.812 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:48.812 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:48.813 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:48.814 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=223), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [23ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [83ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [84ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [84ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [85ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [140ms] refreshing cache from master, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [143ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [221ms] refreshing cache from master, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [222ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [222ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [223ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [223ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@1118149448(state=PENDING, result=null, callback=(continuation of Deferred@1465990267 after retry RPC after error@1625318540) -> (continuation of Deferred@426033064 after retry RPC after error@2012547095) -> (continuation of Deferred@698966134 after retry RPC after error@1242478027) -> (continuation of Deferred@1982581469 after retry RPC after error@692175295) -> (continuation of Deferred@1258925617 after retry RPC after error@1947658870) -> (continuation of Deferred@1547938571 after retry RPC after error@2074658053) -> (continuation of Deferred@2111214695 after retry RPC after error@1053852913), errback=(continuation of Deferred@1465990267 after retry RPC after error@1625318540) -> (continuation of Deferred@426033064 after retry RPC after error@2012547095) -> (continuation of Deferred@698966134 after retry RPC after error@1242478027) -> (continuation of Deferred@1982581469 after retry RPC after error@692175295) -> (continuation of Deferred@1258925617 after retry RPC after error@1947658870) -> (continuation of Deferred@1547938571 after retry RPC after error@2074658053) -> (continuation of Deferred@2111214695 after retry RPC after error@1053852913)))
20:27:48.815 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=223), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [23ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [83ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [84ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [84ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [85ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [140ms] refreshing cache from master, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [143ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [221ms] refreshing cache from master, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [222ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [222ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [223ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [223ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@1118149448(state=PENDING, result=null, callback=(continuation of Deferred@1465990267 after retry RPC after error@1625318540) -> (continuation of Deferred@426033064 after retry RPC after error@2012547095) -> (continuation of Deferred@698966134 after retry RPC after error@1242478027) -> (continuation of Deferred@1982581469 after retry RPC after error@692175295) -> (continuation of Deferred@1258925617 after retry RPC after error@1947658870) -> (continuation of Deferred@1547938571 after retry RPC after error@2074658053) -> (continuation of Deferred@2111214695 after retry RPC after error@1053852913), errback=(continuation of Deferred@1465990267 after retry RPC after error@1625318540) -> (continuation of Deferred@426033064 after retry RPC after error@2012547095) -> (continuation of Deferred@698966134 after retry RPC after error@1242478027) -> (continuation of Deferred@1982581469 after retry RPC after error@692175295) -> (continuation of Deferred@1258925617 after retry RPC after error@1947658870) -> (continuation of Deferred@1547938571 after retry RPC after error@2074658053) -> (continuation of Deferred@2111214695 after retry RPC after error@1053852913)))
20:27:48.831 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:48.831 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:48.832 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:48.832 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:48.832 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:48.832 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:48.832 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:48.832 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:48.832 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:48.851 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:48.852 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:48.852 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:48.852 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:48.852 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:48.852 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:48.852 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:48.853 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:48.853 [WARN - kudu-nio-5] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:48.870 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:48.871 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:48.871 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:48.871 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:48.871 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:48.872 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:48.872 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:48.872 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:48.872 [WARN - kudu-nio-7] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:48.891 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:48.891 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:48.892 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:48.892 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:48.892 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:48.892 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:48.892 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:48.892 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:48.892 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:48.911 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:48.911 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:48.912 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:48.912 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:48.912 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:48.912 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:48.912 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:48.912 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:48.912 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:48.957 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:48.957 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:48.958 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:48.958 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:48.958 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:48.958 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:48.958 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:48.958 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:48.959 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:49.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:49.002979 4288 raft_consensus.cc:493] T 00000000000000000000000000000000 P 0db1d539946942f794b0c4ec81f8f4b4 [term 4 FOLLOWER]: Starting pre-election (detected failure of leader b1731f8308c145f686f88efd6a54b851)
20:27:49.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:49.003298 4288 raft_consensus.cc:515] T 00000000000000000000000000000000 P 0db1d539946942f794b0c4ec81f8f4b4 [term 4 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0db1d539946942f794b0c4ec81f8f4b4" member_type: VOTER last_known_addr { host: "127.3.46.62" port: 46071 } } peers { permanent_uuid: "d6c270b387b04f17b2a58bfcb0eb4f82" member_type: VOTER last_known_addr { host: "127.3.46.61" port: 39407 } } peers { permanent_uuid: "b1731f8308c145f686f88efd6a54b851" member_type: VOTER last_known_addr { host: "127.3.46.60" port: 36127 } }
20:27:49.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:49.004451 4288 leader_election.cc:290] T 00000000000000000000000000000000 P 0db1d539946942f794b0c4ec81f8f4b4 [CANDIDATE]: Term 5 pre-election: Requested pre-vote from peers d6c270b387b04f17b2a58bfcb0eb4f82 (127.3.46.61:39407), b1731f8308c145f686f88efd6a54b851 (127.3.46.60:36127)
20:27:49.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:27:49.005860 4149 leader_election.cc:336] T 00000000000000000000000000000000 P 0db1d539946942f794b0c4ec81f8f4b4 [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer d6c270b387b04f17b2a58bfcb0eb4f82 (127.3.46.61:39407): Network error: Client connection negotiation failed: client connection to 127.3.46.61:39407: connect: Connection refused (error 111)
20:27:49.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:27:49.006443 4148 leader_election.cc:336] T 00000000000000000000000000000000 P 0db1d539946942f794b0c4ec81f8f4b4 [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer b1731f8308c145f686f88efd6a54b851 (127.3.46.60:36127): Network error: Client connection negotiation failed: client connection to 127.3.46.60:36127: connect: Connection refused (error 111)
20:27:49.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:49.006546 4148 leader_election.cc:304] T 00000000000000000000000000000000 P 0db1d539946942f794b0c4ec81f8f4b4 [CANDIDATE]: Term 5 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 0db1d539946942f794b0c4ec81f8f4b4; no voters: b1731f8308c145f686f88efd6a54b851, d6c270b387b04f17b2a58bfcb0eb4f82
20:27:49.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:49.006774 4288 raft_consensus.cc:2749] T 00000000000000000000000000000000 P 0db1d539946942f794b0c4ec81f8f4b4 [term 4 FOLLOWER]: Leader pre-election lost for term 5. Reason: could not achieve majority
20:27:49.010 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:49.011 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:49.012 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:49.012 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:49.012 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:49.013 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:49.013 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:49.013 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:49.013 [WARN - kudu-nio-7] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:49.030 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:49.031 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:49.031 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:49.031 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:49.031 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:49.031 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:49.031 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:49.031 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:49.032 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:49.032 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=201), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [23ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [120ms] refreshing cache from master, [126ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [127ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [127ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [128ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [128ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [129ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [129ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [183ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [183ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@499028947(state=PENDING, result=null, callback=(continuation of Deferred@1740447863 after retry RPC after error@841024807) -> (continuation of Deferred@854698306 after retry RPC after error@1530434756) -> (continuation of Deferred@1426635084 after retry RPC after error@1866110711) -> (continuation of Deferred@1354664427 after retry RPC after error@1337964940) -> (continuation of Deferred@1609411566 after retry RPC after error@1683690442) -> (continuation of Deferred@1083060091 after retry RPC after error@1917820223) -> (continuation of Deferred@1078983235 after retry RPC after error@2092328131), errback=(continuation of Deferred@1740447863 after retry RPC after error@841024807) -> (continuation of Deferred@854698306 after retry RPC after error@1530434756) -> (continuation of Deferred@1426635084 after retry RPC after error@1866110711) -> (continuation of Deferred@1354664427 after retry RPC after error@1337964940) -> (continuation of Deferred@1609411566 after retry RPC after error@1683690442) -> (continuation of Deferred@1083060091 after retry RPC after error@1917820223) -> (continuation of Deferred@1078983235 after retry RPC after error@2092328131)))
20:27:49.033 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=201), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [23ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [120ms] refreshing cache from master, [126ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [127ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [127ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [128ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [128ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [129ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [129ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [183ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [183ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@499028947(state=PENDING, result=null, callback=(continuation of Deferred@1740447863 after retry RPC after error@841024807) -> (continuation of Deferred@854698306 after retry RPC after error@1530434756) -> (continuation of Deferred@1426635084 after retry RPC after error@1866110711) -> (continuation of Deferred@1354664427 after retry RPC after error@1337964940) -> (continuation of Deferred@1609411566 after retry RPC after error@1683690442) -> (continuation of Deferred@1083060091 after retry RPC after error@1917820223) -> (continuation of Deferred@1078983235 after retry RPC after error@2092328131), errback=(continuation of Deferred@1740447863 after retry RPC after error@841024807) -> (continuation of Deferred@854698306 after retry RPC after error@1530434756) -> (continuation of Deferred@1426635084 after retry RPC after error@1866110711) -> (continuation of Deferred@1354664427 after retry RPC after error@1337964940) -> (continuation of Deferred@1609411566 after retry RPC after error@1683690442) -> (continuation of Deferred@1083060091 after retry RPC after error@1917820223) -> (continuation of Deferred@1078983235 after retry RPC after error@2092328131)))
20:27:49.050 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:49.051 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:49.051 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:49.052 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:49.052 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:49.052 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:49.052 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:49.052 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:49.052 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:49.071 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:49.071 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:49.072 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:49.072 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:49.072 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:49.072 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:49.072 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:49.072 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:49.072 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:49.091 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:49.092 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:49.092 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:49.092 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:49.092 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:49.092 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:49.092 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:49.093 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:49.093 [WARN - kudu-nio-7] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:49.110 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:49.111 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:49.111 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:49.111 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:49.111 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:49.112 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:49.112 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:49.112 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:49.112 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:49.131 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:49.131 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:49.132 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:49.132 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:49.132 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:49.132 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:49.132 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:49.132 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:49.132 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:49.151 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:49.151 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:49.152 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:49.152 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:49.152 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:49.152 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:49.152 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:49.152 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:49.153 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:49.190 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:49.191 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:49.192 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:49.192 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:49.192 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:49.192 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:49.192 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:49.192 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:49.192 [WARN - kudu-nio-6] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:49.311 [DEBUG - pool-40-thread-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=260), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [42ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [43ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [43ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [103ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [143ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, deferred=Deferred@34588121(state=PENDING, result=null, callback=(continuation of Deferred@1310517642 after retry RPC after error@707746080) -> (continuation of Deferred@1231183269 after retry RPC after error@901740637) -> (continuation of Deferred@474384345 after retry RPC after error@1290812200) -> (continuation of Deferred@388286566 after retry RPC after error@1494689615) -> (continuation of Deferred@1900485627 after retry RPC after error@1017333632) -> (continuation of Deferred@234213258 after retry RPC after error@1832430628) -> (continuation of Deferred@619313851 after retry RPC after error@47195873), errback=(continuation of Deferred@1310517642 after retry RPC after error@707746080) -> (continuation of Deferred@1231183269 after retry RPC after error@901740637) -> (continuation of Deferred@474384345 after retry RPC after error@1290812200) -> (continuation of Deferred@388286566 after retry RPC after error@1494689615) -> (continuation of Deferred@1900485627 after retry RPC after error@1017333632) -> (continuation of Deferred@234213258 after retry RPC after error@1832430628) -> (continuation of Deferred@619313851 after retry RPC after error@47195873)))
20:27:49.311 [WARN - pool-40-thread-1] (HashedWheelTimer.java:716) An exception was thrown by TimerTask.
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713)
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701)
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:787)
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501)
at java.base/java.lang.Thread.run(Thread.java:840)
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713)
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701)
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:787)
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501)
at java.base/java.lang.Thread.run(Thread.java:840)
20:27:49.330 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:49.331 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:49.331 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:49.331 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:49.331 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:49.331 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:49.331 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:49.332 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:49.332 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:49.350 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:49.351 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:49.351 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:49.351 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:49.351 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:49.351 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:49.352 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:49.352 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:49.352 [WARN - kudu-nio-3] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:49.371 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:49.371 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:49.372 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:49.372 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:49.372 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:49.372 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:49.372 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:49.372 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:49.373 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:49.391 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:49.391 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:49.391 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:49.391 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:49.392 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:49.392 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:49.392 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:49.392 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:49.392 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:49.411 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:49.411 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:49.412 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:49.412 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:49.412 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:49.412 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:49.412 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:49.412 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:49.413 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:49.450 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:49.451 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:49.451 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:49.451 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:49.451 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:49.451 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:49.451 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:49.452 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:49.452 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:49.491 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:49.491 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:49.492 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:49.492 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:49.492 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:49.492 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:49.492 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:49.492 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:49.492 [WARN - kudu-nio-5] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:49.571 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:49.571 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:49.572 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:49.572 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:49.572 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:49.572 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:49.572 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:49.572 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:49.572 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:49.573 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@1098188284(state=PENDING, result=null, callback=(continuation of Deferred@1273367322 after retry RPC after error@193287625) -> (continuation of Deferred@1898388797 after retry RPC after error@404007936) -> (continuation of Deferred@1674693315 after retry RPC after error@1121710189) -> (continuation of Deferred@190628862 after retry RPC after error@915741241) -> (continuation of Deferred@1543461370 after retry RPC after error@1178243696) -> (continuation of Deferred@731183738 after retry RPC after error@902719971) -> (continuation of Deferred@341825065 after retry RPC after error@115638208), errback=(continuation of Deferred@1273367322 after retry RPC after error@193287625) -> (continuation of Deferred@1898388797 after retry RPC after error@404007936) -> (continuation of Deferred@1674693315 after retry RPC after error@1121710189) -> (continuation of Deferred@190628862 after retry RPC after error@915741241) -> (continuation of Deferred@1543461370 after retry RPC after error@1178243696) -> (continuation of Deferred@731183738 after retry RPC after error@902719971) -> (continuation of Deferred@341825065 after retry RPC after error@115638208)))
20:27:49.573 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@1098188284(state=PENDING, result=null, callback=(continuation of Deferred@1273367322 after retry RPC after error@193287625) -> (continuation of Deferred@1898388797 after retry RPC after error@404007936) -> (continuation of Deferred@1674693315 after retry RPC after error@1121710189) -> (continuation of Deferred@190628862 after retry RPC after error@915741241) -> (continuation of Deferred@1543461370 after retry RPC after error@1178243696) -> (continuation of Deferred@731183738 after retry RPC after error@902719971) -> (continuation of Deferred@341825065 after retry RPC after error@115638208), errback=(continuation of Deferred@1273367322 after retry RPC after error@193287625) -> (continuation of Deferred@1898388797 after retry RPC after error@404007936) -> (continuation of Deferred@1674693315 after retry RPC after error@1121710189) -> (continuation of Deferred@190628862 after retry RPC after error@915741241) -> (continuation of Deferred@1543461370 after retry RPC after error@1178243696) -> (continuation of Deferred@731183738 after retry RPC after error@902719971) -> (continuation of Deferred@341825065 after retry RPC after error@115638208)))
20:27:49.591 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:49.591 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:49.592 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:49.592 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:49.592 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:49.592 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:49.592 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:49.592 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:49.592 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:49.610 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:49.611 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:49.611 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:49.611 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:49.611 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:49.611 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:49.611 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:49.611 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:49.612 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:49.630 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:49.631 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:49.631 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:49.631 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:49.631 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:49.632 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:49.632 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:49.632 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:49.632 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:49.651 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:49.651 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:49.652 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:49.652 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:49.652 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:49.652 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:49.652 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:49.652 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:49.652 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:49.671 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:49.671 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:49.672 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:49.672 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:49.672 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:49.672 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:49.672 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:49.672 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:49.673 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:49.690 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:49.691 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:49.691 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:49.691 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:49.691 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:49.691 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:49.691 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:49.691 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:49.692 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:49.731 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:49.731 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:49.732 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:49.732 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:49.732 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:49.732 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:49.732 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:49.732 [INFO - pool-40-thread-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:49.733 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:49.810 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:49.811 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:49.811 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:49.811 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:49.811 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:49.811 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:49.811 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:49.811 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:49.812 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:49.812 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=221), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [20ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [40ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [99ms] refreshing cache from master, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [100ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [100ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [101ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [139ms] refreshing cache from master, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [219ms] refreshing cache from master, [219ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [219ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [220ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [220ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [221ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@1906091159(state=PENDING, result=null, callback=(continuation of Deferred@2061755668 after retry RPC after error@949535227) -> (continuation of Deferred@604612824 after retry RPC after error@816126162) -> (continuation of Deferred@1435205783 after retry RPC after error@727844036) -> (continuation of Deferred@1120151064 after retry RPC after error@1514132762) -> (continuation of Deferred@1404660210 after retry RPC after error@1757807461) -> (continuation of Deferred@198253699 after retry RPC after error@803381222) -> (continuation of Deferred@931960081 after retry RPC after error@1862890926), errback=(continuation of Deferred@2061755668 after retry RPC after error@949535227) -> (continuation of Deferred@604612824 after retry RPC after error@816126162) -> (continuation of Deferred@1435205783 after retry RPC after error@727844036) -> (continuation of Deferred@1120151064 after retry RPC after error@1514132762) -> (continuation of Deferred@1404660210 after retry RPC after error@1757807461) -> (continuation of Deferred@198253699 after retry RPC after error@803381222) -> (continuation of Deferred@931960081 after retry RPC after error@1862890926)))
20:27:49.813 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=221), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [20ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [40ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [99ms] refreshing cache from master, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [100ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [100ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [101ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [139ms] refreshing cache from master, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [219ms] refreshing cache from master, [219ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [219ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [220ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [220ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [221ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@1906091159(state=PENDING, result=null, callback=(continuation of Deferred@2061755668 after retry RPC after error@949535227) -> (continuation of Deferred@604612824 after retry RPC after error@816126162) -> (continuation of Deferred@1435205783 after retry RPC after error@727844036) -> (continuation of Deferred@1120151064 after retry RPC after error@1514132762) -> (continuation of Deferred@1404660210 after retry RPC after error@1757807461) -> (continuation of Deferred@198253699 after retry RPC after error@803381222) -> (continuation of Deferred@931960081 after retry RPC after error@1862890926), errback=(continuation of Deferred@2061755668 after retry RPC after error@949535227) -> (continuation of Deferred@604612824 after retry RPC after error@816126162) -> (continuation of Deferred@1435205783 after retry RPC after error@727844036) -> (continuation of Deferred@1120151064 after retry RPC after error@1514132762) -> (continuation of Deferred@1404660210 after retry RPC after error@1757807461) -> (continuation of Deferred@198253699 after retry RPC after error@803381222) -> (continuation of Deferred@931960081 after retry RPC after error@1862890926)))
20:27:49.830 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:49.831 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:49.832 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:49.832 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:49.832 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:49.832 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:49.832 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:49.832 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:49.832 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:49.851 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:49.852 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:49.852 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:49.852 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:49.852 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:49.852 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:49.852 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:49.852 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:49.853 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:49.870 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:49.871 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:49.871 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:49.871 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:49.872 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:49.872 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:49.872 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:49.872 [INFO - pool-40-thread-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:49.872 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:49.890 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:49.891 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:49.891 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:49.891 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:49.891 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:49.891 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:49.892 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:49.892 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:49.892 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:49.911 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:49.911 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:49.912 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:49.912 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:49.912 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:49.912 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:49.912 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:49.912 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:49.913 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:49.931 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:49.931 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:49.932 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:49.932 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:49.932 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:49.932 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:49.932 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:49.932 [INFO - pool-40-thread-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:49.933 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:49.970 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:49.971 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:49.971 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:49.971 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:49.971 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:49.972 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:49.973 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:49.973 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:49.973 [WARN - kudu-nio-5] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:49.991 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:49.991 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:49.992 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:49.992 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:49.992 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:49.992 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:49.992 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:49.992 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:49.992 [WARN - kudu-nio-7] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:49.993 [DEBUG - kudu-nio-7] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [103ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [143ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [143ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@674385857(state=PENDING, result=null, callback=(continuation of Deferred@1508060281 after retry RPC after error@611852795) -> (continuation of Deferred@789214087 after retry RPC after error@368868832) -> (continuation of Deferred@1438387138 after retry RPC after error@816746359) -> (continuation of Deferred@1279546542 after retry RPC after error@975254459) -> (continuation of Deferred@259444006 after retry RPC after error@1273446024) -> (continuation of Deferred@627630718 after retry RPC after error@384278915) -> (continuation of Deferred@801198468 after retry RPC after error@1562388874), errback=(continuation of Deferred@1508060281 after retry RPC after error@611852795) -> (continuation of Deferred@789214087 after retry RPC after error@368868832) -> (continuation of Deferred@1438387138 after retry RPC after error@816746359) -> (continuation of Deferred@1279546542 after retry RPC after error@975254459) -> (continuation of Deferred@259444006 after retry RPC after error@1273446024) -> (continuation of Deferred@627630718 after retry RPC after error@384278915) -> (continuation of Deferred@801198468 after retry RPC after error@1562388874)))
20:27:49.993 [DEBUG - kudu-nio-7] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [103ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [143ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [143ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@674385857(state=PENDING, result=null, callback=(continuation of Deferred@1508060281 after retry RPC after error@611852795) -> (continuation of Deferred@789214087 after retry RPC after error@368868832) -> (continuation of Deferred@1438387138 after retry RPC after error@816746359) -> (continuation of Deferred@1279546542 after retry RPC after error@975254459) -> (continuation of Deferred@259444006 after retry RPC after error@1273446024) -> (continuation of Deferred@627630718 after retry RPC after error@384278915) -> (continuation of Deferred@801198468 after retry RPC after error@1562388874), errback=(continuation of Deferred@1508060281 after retry RPC after error@611852795) -> (continuation of Deferred@789214087 after retry RPC after error@368868832) -> (continuation of Deferred@1438387138 after retry RPC after error@816746359) -> (continuation of Deferred@1279546542 after retry RPC after error@975254459) -> (continuation of Deferred@259444006 after retry RPC after error@1273446024) -> (continuation of Deferred@627630718 after retry RPC after error@384278915) -> (continuation of Deferred@801198468 after retry RPC after error@1562388874)))
20:27:50.011 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:50.011 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:50.012 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:50.012 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:50.012 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:50.012 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:50.012 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:50.012 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:50.013 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:50.031 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:50.032 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:50.032 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:50.032 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:50.032 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:50.032 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:50.032 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:50.032 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:50.033 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:50.050 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:50.051 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:50.051 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:50.051 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:50.051 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:50.052 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:50.052 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:50.052 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:50.052 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:50.071 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:50.071 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:50.072 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:50.072 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:50.072 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:50.072 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:50.072 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:50.072 [INFO - pool-40-thread-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:50.072 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:50.091 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:50.091 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:50.092 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:50.092 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:50.092 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:50.092 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:50.092 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:50.092 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:50.093 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:50.130 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:50.131 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:50.131 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:50.131 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:50.131 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:50.132 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:50.132 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:50.132 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:50.132 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:50.171 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:50.171 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:50.172 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:50.172 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:50.172 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:50.172 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:50.172 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:50.172 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:50.173 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:50.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:50.186045 4132 master_runner.cc:186] Error getting master registration for 127.3.46.61:39407: Network error: Client connection negotiation failed: client connection to 127.3.46.61:39407: connect: Connection refused (error 111),
20:27:50.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:50.186225 4132 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
20:27:50.271 [DEBUG - pool-40-thread-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=259), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [79ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [119ms] refreshing cache from master, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [120ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [121ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [159ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@1518669769(state=PENDING, result=null, callback=(continuation of Deferred@1467616747 after retry RPC after error@1714406556) -> (continuation of Deferred@301553308 after retry RPC after error@1668347949) -> (continuation of Deferred@1684186365 after retry RPC after error@2096207817) -> (continuation of Deferred@669475709 after retry RPC after error@1127368252) -> (continuation of Deferred@847661035 after retry RPC after error@1111081389) -> (continuation of Deferred@1591328284 after retry RPC after error@252383325) -> (continuation of Deferred@254977864 after retry RPC after error@59390890), errback=(continuation of Deferred@1467616747 after retry RPC after error@1714406556) -> (continuation of Deferred@301553308 after retry RPC after error@1668347949) -> (continuation of Deferred@1684186365 after retry RPC after error@2096207817) -> (continuation of Deferred@669475709 after retry RPC after error@1127368252) -> (continuation of Deferred@847661035 after retry RPC after error@1111081389) -> (continuation of Deferred@1591328284 after retry RPC after error@252383325) -> (continuation of Deferred@254977864 after retry RPC after error@59390890)))
20:27:50.271 [WARN - pool-40-thread-1] (HashedWheelTimer.java:716) An exception was thrown by TimerTask.
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713)
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701)
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:787)
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501)
at java.base/java.lang.Thread.run(Thread.java:840)
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713)
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701)
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:787)
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501)
at java.base/java.lang.Thread.run(Thread.java:840)
20:27:50.290 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:50.291 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:50.291 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:50.291 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:50.291 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:50.291 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:50.291 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:50.291 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:50.292 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:50.310 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:50.311 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:50.311 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:50.311 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:50.311 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:50.312 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:50.312 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:50.312 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:50.312 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:50.331 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:50.331 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:50.331 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:50.332 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:50.332 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:50.332 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:50.332 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:50.332 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:50.332 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:50.351 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:50.351 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:50.352 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:50.352 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:50.352 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:50.352 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:50.352 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:50.352 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:50.352 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:50.370 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:50.371 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:50.371 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:50.371 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:50.371 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:50.372 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:50.372 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:50.372 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:50.372 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:50.411 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:50.411 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:50.412 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:50.412 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:50.412 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:50.412 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:50.412 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:50.412 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:50.412 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:50.431 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:50.432 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:50.432 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:50.432 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:50.432 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:50.432 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:50.432 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:50.432 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:50.433 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:50.531 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:50.531 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:50.532 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:50.532 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:50.532 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:50.532 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:50.532 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:50.532 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:50.533 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:50.533 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=243), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [141ms] refreshing cache from master, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [142ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [241ms] refreshing cache from master, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [243ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [243ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@1952647477(state=PENDING, result=null, callback=(continuation of Deferred@928915141 after retry RPC after error@162898771) -> (continuation of Deferred@26432577 after retry RPC after error@1834340327) -> (continuation of Deferred@716856510 after retry RPC after error@238144897) -> (continuation of Deferred@354132314 after retry RPC after error@1458732777) -> (continuation of Deferred@777361189 after retry RPC after error@780667563) -> (continuation of Deferred@1716617041 after retry RPC after error@588333124) -> (continuation of Deferred@561977276 after retry RPC after error@1330416746), errback=(continuation of Deferred@928915141 after retry RPC after error@162898771) -> (continuation of Deferred@26432577 after retry RPC after error@1834340327) -> (continuation of Deferred@716856510 after retry RPC after error@238144897) -> (continuation of Deferred@354132314 after retry RPC after error@1458732777) -> (continuation of Deferred@777361189 after retry RPC after error@780667563) -> (continuation of Deferred@1716617041 after retry RPC after error@588333124) -> (continuation of Deferred@561977276 after retry RPC after error@1330416746)))
20:27:50.534 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=243), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [141ms] refreshing cache from master, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [142ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [241ms] refreshing cache from master, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [243ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [243ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@1952647477(state=PENDING, result=null, callback=(continuation of Deferred@928915141 after retry RPC after error@162898771) -> (continuation of Deferred@26432577 after retry RPC after error@1834340327) -> (continuation of Deferred@716856510 after retry RPC after error@238144897) -> (continuation of Deferred@354132314 after retry RPC after error@1458732777) -> (continuation of Deferred@777361189 after retry RPC after error@780667563) -> (continuation of Deferred@1716617041 after retry RPC after error@588333124) -> (continuation of Deferred@561977276 after retry RPC after error@1330416746), errback=(continuation of Deferred@928915141 after retry RPC after error@162898771) -> (continuation of Deferred@26432577 after retry RPC after error@1834340327) -> (continuation of Deferred@716856510 after retry RPC after error@238144897) -> (continuation of Deferred@354132314 after retry RPC after error@1458732777) -> (continuation of Deferred@777361189 after retry RPC after error@780667563) -> (continuation of Deferred@1716617041 after retry RPC after error@588333124) -> (continuation of Deferred@561977276 after retry RPC after error@1330416746)))
20:27:50.551 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:50.552 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:50.552 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:50.552 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:50.552 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:50.552 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:50.552 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:50.552 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:50.552 [WARN - kudu-nio-7] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:50.570 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:50.571 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:50.571 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:50.571 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:50.571 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:50.571 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:50.572 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:50.572 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:50.572 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:50.591 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:50.591 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:50.592 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:50.592 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:50.592 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:50.592 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:50.592 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:50.592 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:50.592 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:50.611 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:50.611 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:50.612 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:50.612 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:50.612 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:50.612 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:50.612 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:50.612 [INFO - pool-40-thread-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:50.612 [WARN - pool-40-thread-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:50.631 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:50.631 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:50.632 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:50.632 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:50.632 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:50.632 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:50.632 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:50.632 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:50.633 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:50.670 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:50.671 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:50.671 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:50.671 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:50.671 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:50.672 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:50.672 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:50.672 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:50.672 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:50.711 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:50.711 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:50.712 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:50.712 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:50.712 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:50.712 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:50.712 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:50.712 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:50.712 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:50.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:50.781625 4303 raft_consensus.cc:493] T 00000000000000000000000000000000 P 0db1d539946942f794b0c4ec81f8f4b4 [term 4 FOLLOWER]: Starting pre-election (detected failure of leader b1731f8308c145f686f88efd6a54b851)
20:27:50.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:50.781936 4303 raft_consensus.cc:515] T 00000000000000000000000000000000 P 0db1d539946942f794b0c4ec81f8f4b4 [term 4 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0db1d539946942f794b0c4ec81f8f4b4" member_type: VOTER last_known_addr { host: "127.3.46.62" port: 46071 } } peers { permanent_uuid: "d6c270b387b04f17b2a58bfcb0eb4f82" member_type: VOTER last_known_addr { host: "127.3.46.61" port: 39407 } } peers { permanent_uuid: "b1731f8308c145f686f88efd6a54b851" member_type: VOTER last_known_addr { host: "127.3.46.60" port: 36127 } }
20:27:50.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:50.782724 4303 leader_election.cc:290] T 00000000000000000000000000000000 P 0db1d539946942f794b0c4ec81f8f4b4 [CANDIDATE]: Term 5 pre-election: Requested pre-vote from peers d6c270b387b04f17b2a58bfcb0eb4f82 (127.3.46.61:39407), b1731f8308c145f686f88efd6a54b851 (127.3.46.60:36127)
20:27:50.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:27:50.783715 4148 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.3.46.60:36127: connect: Connection refused (error 111) [suppressed 10 similar messages]
20:27:50.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:27:50.785202 4149 leader_election.cc:336] T 00000000000000000000000000000000 P 0db1d539946942f794b0c4ec81f8f4b4 [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer d6c270b387b04f17b2a58bfcb0eb4f82 (127.3.46.61:39407): Network error: Client connection negotiation failed: client connection to 127.3.46.61:39407: connect: Connection refused (error 111)
20:27:50.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:27:50.785398 4148 leader_election.cc:336] T 00000000000000000000000000000000 P 0db1d539946942f794b0c4ec81f8f4b4 [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer b1731f8308c145f686f88efd6a54b851 (127.3.46.60:36127): Network error: Client connection negotiation failed: client connection to 127.3.46.60:36127: connect: Connection refused (error 111)
20:27:50.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:50.785527 4148 leader_election.cc:304] T 00000000000000000000000000000000 P 0db1d539946942f794b0c4ec81f8f4b4 [CANDIDATE]: Term 5 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 0db1d539946942f794b0c4ec81f8f4b4; no voters: b1731f8308c145f686f88efd6a54b851, d6c270b387b04f17b2a58bfcb0eb4f82
20:27:50.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:50.785775 4303 raft_consensus.cc:2749] T 00000000000000000000000000000000 P 0db1d539946942f794b0c4ec81f8f4b4 [term 4 FOLLOWER]: Leader pre-election lost for term 5. Reason: could not achieve majority
20:27:50.790 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:50.791 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:50.791 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:50.791 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:50.791 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:50.791 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:50.791 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:50.791 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:50.792 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:50.792 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=241), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [20ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [59ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [119ms] refreshing cache from master, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [120ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [121ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [159ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [161ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [239ms] refreshing cache from master, [239ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [239ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [240ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [240ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [241ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@219696767(state=PENDING, result=null, callback=(continuation of Deferred@64267042 after retry RPC after error@228768071) -> (continuation of Deferred@252905598 after retry RPC after error@1077400661) -> (continuation of Deferred@1717347010 after retry RPC after error@2116835279) -> (continuation of Deferred@1777066124 after retry RPC after error@394265752) -> (continuation of Deferred@348778008 after retry RPC after error@147276129) -> (continuation of Deferred@1042165621 after retry RPC after error@690833642) -> (continuation of Deferred@1917831929 after retry RPC after error@1085358488), errback=(continuation of Deferred@64267042 after retry RPC after error@228768071) -> (continuation of Deferred@252905598 after retry RPC after error@1077400661) -> (continuation of Deferred@1717347010 after retry RPC after error@2116835279) -> (continuation of Deferred@1777066124 after retry RPC after error@394265752) -> (continuation of Deferred@348778008 after retry RPC after error@147276129) -> (continuation of Deferred@1042165621 after retry RPC after error@690833642) -> (continuation of Deferred@1917831929 after retry RPC after error@1085358488)))
20:27:50.792 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=241), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [20ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [59ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [119ms] refreshing cache from master, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [120ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [121ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [159ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [161ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [239ms] refreshing cache from master, [239ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [239ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [240ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [240ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [241ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@219696767(state=PENDING, result=null, callback=(continuation of Deferred@64267042 after retry RPC after error@228768071) -> (continuation of Deferred@252905598 after retry RPC after error@1077400661) -> (continuation of Deferred@1717347010 after retry RPC after error@2116835279) -> (continuation of Deferred@1777066124 after retry RPC after error@394265752) -> (continuation of Deferred@348778008 after retry RPC after error@147276129) -> (continuation of Deferred@1042165621 after retry RPC after error@690833642) -> (continuation of Deferred@1917831929 after retry RPC after error@1085358488), errback=(continuation of Deferred@64267042 after retry RPC after error@228768071) -> (continuation of Deferred@252905598 after retry RPC after error@1077400661) -> (continuation of Deferred@1717347010 after retry RPC after error@2116835279) -> (continuation of Deferred@1777066124 after retry RPC after error@394265752) -> (continuation of Deferred@348778008 after retry RPC after error@147276129) -> (continuation of Deferred@1042165621 after retry RPC after error@690833642) -> (continuation of Deferred@1917831929 after retry RPC after error@1085358488)))
20:27:50.811 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:50.811 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:50.812 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:50.812 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:50.812 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:50.812 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:50.812 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:50.812 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:50.812 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:50.831 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:50.832 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:50.832 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:50.832 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:50.832 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:50.832 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:50.832 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:50.832 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:50.833 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:50.850 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:50.851 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:50.851 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:50.851 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:50.851 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:50.851 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:50.852 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:50.852 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:50.852 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:50.871 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:50.871 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:50.871 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:50.872 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:50.872 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:50.872 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:50.872 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:50.872 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:50.872 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:50.891 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:50.891 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:50.892 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:50.892 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:50.892 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:50.892 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:50.892 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:50.892 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:50.892 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:50.930 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:50.931 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:50.931 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:50.931 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:50.931 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:50.931 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:50.932 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:50.932 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:50.932 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:50.991 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:50.991 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:50.992 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:50.992 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:50.992 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:50.992 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:50.992 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:50.992 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:50.992 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:50.993 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [180ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@1489237344(state=PENDING, result=null, callback=(continuation of Deferred@48807861 after retry RPC after error@1960843827) -> (continuation of Deferred@1900821029 after retry RPC after error@920889753) -> (continuation of Deferred@1907473895 after retry RPC after error@99881056) -> (continuation of Deferred@360288233 after retry RPC after error@1048400830) -> (continuation of Deferred@1113558803 after retry RPC after error@1728962600) -> (continuation of Deferred@1045573095 after retry RPC after error@1785096496), errback=(continuation of Deferred@48807861 after retry RPC after error@1960843827) -> (continuation of Deferred@1900821029 after retry RPC after error@920889753) -> (continuation of Deferred@1907473895 after retry RPC after error@99881056) -> (continuation of Deferred@360288233 after retry RPC after error@1048400830) -> (continuation of Deferred@1113558803 after retry RPC after error@1728962600) -> (continuation of Deferred@1045573095 after retry RPC after error@1785096496)))
20:27:50.993 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [180ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@1489237344(state=PENDING, result=null, callback=(continuation of Deferred@48807861 after retry RPC after error@1960843827) -> (continuation of Deferred@1900821029 after retry RPC after error@920889753) -> (continuation of Deferred@1907473895 after retry RPC after error@99881056) -> (continuation of Deferred@360288233 after retry RPC after error@1048400830) -> (continuation of Deferred@1113558803 after retry RPC after error@1728962600) -> (continuation of Deferred@1045573095 after retry RPC after error@1785096496), errback=(continuation of Deferred@48807861 after retry RPC after error@1960843827) -> (continuation of Deferred@1900821029 after retry RPC after error@920889753) -> (continuation of Deferred@1907473895 after retry RPC after error@99881056) -> (continuation of Deferred@360288233 after retry RPC after error@1048400830) -> (continuation of Deferred@1113558803 after retry RPC after error@1728962600) -> (continuation of Deferred@1045573095 after retry RPC after error@1785096496)))
20:27:51.011 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:51.011 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:51.011 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:51.012 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:51.012 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:51.012 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:51.012 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:51.012 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:51.013 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:51.031 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:51.031 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:51.032 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:51.032 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:51.032 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:51.032 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:51.032 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:51.032 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:51.032 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:51.051 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:51.052 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:51.052 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:51.052 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:51.052 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:51.052 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:51.052 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:51.052 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:51.053 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:51.070 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:51.071 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:51.071 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:51.071 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:51.071 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:51.071 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:51.071 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:51.071 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:51.072 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:51.090 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:51.091 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:51.091 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:51.092 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:51.092 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:51.092 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:51.092 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:51.092 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:51.092 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:51.111 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:51.111 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:51.111 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:51.112 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:51.112 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:51.112 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:51.112 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:51.112 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:51.112 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:51.131 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:51.132 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:51.132 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:51.132 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:51.132 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:51.132 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:51.132 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:51.132 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:51.133 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:51.191 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:51.191 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:51.191 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:51.192 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:51.192 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:51.192 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:51.192 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:51.192 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:51.192 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:51.193 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [3ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [43ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [123ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@82658329(state=PENDING, result=null, callback=(continuation of Deferred@126232177 after retry RPC after error@979015250) -> (continuation of Deferred@1759148550 after retry RPC after error@116995720) -> (continuation of Deferred@813317512 after retry RPC after error@1593624019) -> (continuation of Deferred@1699585600 after retry RPC after error@709863703) -> (continuation of Deferred@1270368171 after retry RPC after error@1934803282) -> (continuation of Deferred@2061823718 after retry RPC after error@766834620), errback=(continuation of Deferred@126232177 after retry RPC after error@979015250) -> (continuation of Deferred@1759148550 after retry RPC after error@116995720) -> (continuation of Deferred@813317512 after retry RPC after error@1593624019) -> (continuation of Deferred@1699585600 after retry RPC after error@709863703) -> (continuation of Deferred@1270368171 after retry RPC after error@1934803282) -> (continuation of Deferred@2061823718 after retry RPC after error@766834620)))
20:27:51.193 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [3ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [43ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [123ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@82658329(state=PENDING, result=null, callback=(continuation of Deferred@126232177 after retry RPC after error@979015250) -> (continuation of Deferred@1759148550 after retry RPC after error@116995720) -> (continuation of Deferred@813317512 after retry RPC after error@1593624019) -> (continuation of Deferred@1699585600 after retry RPC after error@709863703) -> (continuation of Deferred@1270368171 after retry RPC after error@1934803282) -> (continuation of Deferred@2061823718 after retry RPC after error@766834620), errback=(continuation of Deferred@126232177 after retry RPC after error@979015250) -> (continuation of Deferred@1759148550 after retry RPC after error@116995720) -> (continuation of Deferred@813317512 after retry RPC after error@1593624019) -> (continuation of Deferred@1699585600 after retry RPC after error@709863703) -> (continuation of Deferred@1270368171 after retry RPC after error@1934803282) -> (continuation of Deferred@2061823718 after retry RPC after error@766834620)))
20:27:51.211 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:51.211 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:51.212 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:51.212 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:51.212 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:51.212 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:51.212 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:51.212 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:51.212 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:51.230 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:51.231 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:51.231 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:51.231 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:51.231 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:51.231 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:51.231 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:51.231 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:51.232 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:51.250 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:51.251 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:51.251 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:51.251 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:51.252 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:51.252 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:51.252 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:51.252 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:51.252 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:51.271 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:51.271 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:51.272 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:51.272 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:51.272 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:51.272 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:51.272 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:51.272 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:51.272 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:51.291 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:51.291 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:51.292 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:51.292 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:51.292 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:51.292 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:51.292 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:51.292 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:51.292 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:51.310 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:51.310 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:51.311 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:51.311 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:51.311 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:51.311 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:51.311 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:51.311 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:51.312 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:51.370 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:51.371 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:51.371 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:51.371 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:51.371 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:51.372 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:51.372 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:51.372 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:51.372 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:51.391 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:51.391 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:51.392 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:51.392 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:51.392 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:51.392 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:51.392 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:51.392 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:51.392 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:51.411 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:51.411 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:51.412 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:51.412 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:51.412 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:51.412 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:51.412 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:51.412 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:51.413 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:51.413 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [201ms] refreshing cache from master, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [203ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@1909611247(state=PENDING, result=null, callback=(continuation of Deferred@454348250 after retry RPC after error@1578841130) -> (continuation of Deferred@166470340 after retry RPC after error@1787429940) -> (continuation of Deferred@1473080994 after retry RPC after error@1349564949) -> (continuation of Deferred@954353885 after retry RPC after error@41107537) -> (continuation of Deferred@1056409283 after retry RPC after error@1803694874) -> (continuation of Deferred@1091316247 after retry RPC after error@1508215727) -> (continuation of Deferred@1537662926 after retry RPC after error@558251331) -> (continuation of Deferred@29030382 after retry RPC after error@1954000005), errback=(continuation of Deferred@454348250 after retry RPC after error@1578841130) -> (continuation of Deferred@166470340 after retry RPC after error@1787429940) -> (continuation of Deferred@1473080994 after retry RPC after error@1349564949) -> (continuation of Deferred@954353885 after retry RPC after error@41107537) -> (continuation of Deferred@1056409283 after retry RPC after error@1803694874) -> (continuation of Deferred@1091316247 after retry RPC after error@1508215727) -> (continuation of Deferred@1537662926 after retry RPC after error@558251331) -> (continuation of Deferred@29030382 after retry RPC after error@1954000005)))
20:27:51.413 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [201ms] refreshing cache from master, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [203ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@1909611247(state=PENDING, result=null, callback=(continuation of Deferred@454348250 after retry RPC after error@1578841130) -> (continuation of Deferred@166470340 after retry RPC after error@1787429940) -> (continuation of Deferred@1473080994 after retry RPC after error@1349564949) -> (continuation of Deferred@954353885 after retry RPC after error@41107537) -> (continuation of Deferred@1056409283 after retry RPC after error@1803694874) -> (continuation of Deferred@1091316247 after retry RPC after error@1508215727) -> (continuation of Deferred@1537662926 after retry RPC after error@558251331) -> (continuation of Deferred@29030382 after retry RPC after error@1954000005), errback=(continuation of Deferred@454348250 after retry RPC after error@1578841130) -> (continuation of Deferred@166470340 after retry RPC after error@1787429940) -> (continuation of Deferred@1473080994 after retry RPC after error@1349564949) -> (continuation of Deferred@954353885 after retry RPC after error@41107537) -> (continuation of Deferred@1056409283 after retry RPC after error@1803694874) -> (continuation of Deferred@1091316247 after retry RPC after error@1508215727) -> (continuation of Deferred@1537662926 after retry RPC after error@558251331) -> (continuation of Deferred@29030382 after retry RPC after error@1954000005)))
20:27:51.430 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:51.431 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:51.431 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:51.431 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:51.431 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:51.431 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:51.431 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:51.431 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:51.432 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:51.450 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:51.451 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:51.451 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:51.451 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:51.451 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:51.451 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:51.452 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:51.452 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:51.452 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:51.471 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:51.471 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:51.471 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:51.471 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:51.471 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:51.472 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:51.472 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:51.472 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:51.472 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:51.491 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:51.491 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:51.492 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:51.492 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:51.492 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:51.492 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:51.492 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:51.492 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:51.492 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:51.511 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:51.512 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:51.512 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:51.512 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:51.512 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:51.512 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:51.512 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:51.512 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:51.513 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:51.530 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:51.531 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:51.531 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:51.531 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:51.531 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:51.531 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:51.532 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:51.532 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:51.532 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:51.571 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:51.571 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:51.571 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:51.572 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:51.572 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:51.572 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:51.572 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:51.572 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:51.572 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:51.611 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:51.612 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:51.612 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:51.612 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:51.612 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:51.612 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:51.612 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:51.612 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:51.613 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:51.613 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [83ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [181ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [183ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@496315943(state=PENDING, result=null, callback=(continuation of Deferred@1651328462 after retry RPC after error@1461111597) -> (continuation of Deferred@2135769657 after retry RPC after error@1260240865) -> (continuation of Deferred@1138151545 after retry RPC after error@586691725) -> (continuation of Deferred@934405729 after retry RPC after error@292926913) -> (continuation of Deferred@1691744910 after retry RPC after error@124446405) -> (continuation of Deferred@1062242983 after retry RPC after error@54000433) -> (continuation of Deferred@1216180307 after retry RPC after error@410118999), errback=(continuation of Deferred@1651328462 after retry RPC after error@1461111597) -> (continuation of Deferred@2135769657 after retry RPC after error@1260240865) -> (continuation of Deferred@1138151545 after retry RPC after error@586691725) -> (continuation of Deferred@934405729 after retry RPC after error@292926913) -> (continuation of Deferred@1691744910 after retry RPC after error@124446405) -> (continuation of Deferred@1062242983 after retry RPC after error@54000433) -> (continuation of Deferred@1216180307 after retry RPC after error@410118999)))
20:27:51.613 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [83ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [181ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [183ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@496315943(state=PENDING, result=null, callback=(continuation of Deferred@1651328462 after retry RPC after error@1461111597) -> (continuation of Deferred@2135769657 after retry RPC after error@1260240865) -> (continuation of Deferred@1138151545 after retry RPC after error@586691725) -> (continuation of Deferred@934405729 after retry RPC after error@292926913) -> (continuation of Deferred@1691744910 after retry RPC after error@124446405) -> (continuation of Deferred@1062242983 after retry RPC after error@54000433) -> (continuation of Deferred@1216180307 after retry RPC after error@410118999), errback=(continuation of Deferred@1651328462 after retry RPC after error@1461111597) -> (continuation of Deferred@2135769657 after retry RPC after error@1260240865) -> (continuation of Deferred@1138151545 after retry RPC after error@586691725) -> (continuation of Deferred@934405729 after retry RPC after error@292926913) -> (continuation of Deferred@1691744910 after retry RPC after error@124446405) -> (continuation of Deferred@1062242983 after retry RPC after error@54000433) -> (continuation of Deferred@1216180307 after retry RPC after error@410118999)))
20:27:51.630 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:51.631 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:51.631 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:51.631 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:51.631 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:51.631 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:51.631 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:51.631 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:51.632 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:51.650 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:51.651 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:51.651 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:51.651 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:51.651 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:51.651 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:51.651 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:51.651 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:51.652 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:51.670 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:51.671 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:51.671 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:51.671 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:51.672 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:51.672 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:51.672 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:51.672 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:51.672 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:51.691 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:51.691 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:51.692 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:51.692 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:51.692 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:51.692 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:51.692 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:51.692 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:51.692 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:51.711 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:51.712 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:51.712 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:51.712 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:51.712 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:51.713 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:51.713 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:51.713 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:51.713 [WARN - kudu-nio-7] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:51.751 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:51.751 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:51.752 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:51.752 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:51.752 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:51.752 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:51.752 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:51.752 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:51.753 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:51.810 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:51.811 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:51.811 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:51.811 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:51.811 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:51.811 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:51.811 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:51.812 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:51.812 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:51.812 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [83ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [120ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [123ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@1913725631(state=PENDING, result=null, callback=(continuation of Deferred@55503972 after retry RPC after error@825005576) -> (continuation of Deferred@2004530619 after retry RPC after error@201684273) -> (continuation of Deferred@2027717913 after retry RPC after error@764788956) -> (continuation of Deferred@257978296 after retry RPC after error@678113176) -> (continuation of Deferred@872649665 after retry RPC after error@932394685) -> (continuation of Deferred@990024280 after retry RPC after error@1248095567), errback=(continuation of Deferred@55503972 after retry RPC after error@825005576) -> (continuation of Deferred@2004530619 after retry RPC after error@201684273) -> (continuation of Deferred@2027717913 after retry RPC after error@764788956) -> (continuation of Deferred@257978296 after retry RPC after error@678113176) -> (continuation of Deferred@872649665 after retry RPC after error@932394685) -> (continuation of Deferred@990024280 after retry RPC after error@1248095567)))
20:27:51.813 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [83ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [120ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [123ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@1913725631(state=PENDING, result=null, callback=(continuation of Deferred@55503972 after retry RPC after error@825005576) -> (continuation of Deferred@2004530619 after retry RPC after error@201684273) -> (continuation of Deferred@2027717913 after retry RPC after error@764788956) -> (continuation of Deferred@257978296 after retry RPC after error@678113176) -> (continuation of Deferred@872649665 after retry RPC after error@932394685) -> (continuation of Deferred@990024280 after retry RPC after error@1248095567), errback=(continuation of Deferred@55503972 after retry RPC after error@825005576) -> (continuation of Deferred@2004530619 after retry RPC after error@201684273) -> (continuation of Deferred@2027717913 after retry RPC after error@764788956) -> (continuation of Deferred@257978296 after retry RPC after error@678113176) -> (continuation of Deferred@872649665 after retry RPC after error@932394685) -> (continuation of Deferred@990024280 after retry RPC after error@1248095567)))
20:27:51.831 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:51.831 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:51.832 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:51.832 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:51.832 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:51.832 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:51.832 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:51.832 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:51.832 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:51.851 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:51.851 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:51.852 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:51.852 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:51.852 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:51.852 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:51.852 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:51.852 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:51.852 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:51.871 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:51.872 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:51.872 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:51.872 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:51.872 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:51.872 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:51.872 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:51.873 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:51.873 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:51.891 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:51.891 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:51.892 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:51.892 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:51.892 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:51.892 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:51.892 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:51.892 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:51.892 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:51.911 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:51.912 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:51.912 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:51.913 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:51.913 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:51.913 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:51.913 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:51.913 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:51.913 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:51.930 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:51.931 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:51.931 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:51.931 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:51.931 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:51.931 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:51.931 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:51.932 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:51.932 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:51.951 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:51.951 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:51.952 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:51.952 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:51.952 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:51.952 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:51.952 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:51.952 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:51.952 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:51.991 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:51.992 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:51.992 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:51.992 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:51.992 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:51.992 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:51.992 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:51.993 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:51.993 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:52.031 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:52.031 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:52.031 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:52.031 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:52.031 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:52.032 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:52.032 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:52.032 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:52.032 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:52.033 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [20ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [43ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [82ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [83ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [83ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [83ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [161ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [162ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [163ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [163ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@840503851(state=PENDING, result=null, callback=(continuation of Deferred@852402525 after retry RPC after error@214437234) -> (continuation of Deferred@1635202294 after retry RPC after error@453372601) -> (continuation of Deferred@1543922893 after retry RPC after error@283722146) -> (continuation of Deferred@1214318541 after retry RPC after error@2080730202) -> (continuation of Deferred@332029723 after retry RPC after error@1240787089) -> (continuation of Deferred@158056717 after retry RPC after error@1591417798) -> (continuation of Deferred@531150209 after retry RPC after error@1903588997) -> (continuation of Deferred@641901071 after retry RPC after error@541169706), errback=(continuation of Deferred@852402525 after retry RPC after error@214437234) -> (continuation of Deferred@1635202294 after retry RPC after error@453372601) -> (continuation of Deferred@1543922893 after retry RPC after error@283722146) -> (continuation of Deferred@1214318541 after retry RPC after error@2080730202) -> (continuation of Deferred@332029723 after retry RPC after error@1240787089) -> (continuation of Deferred@158056717 after retry RPC after error@1591417798) -> (continuation of Deferred@531150209 after retry RPC after error@1903588997) -> (continuation of Deferred@641901071 after retry RPC after error@541169706)))
20:27:52.033 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [20ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [43ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [82ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [83ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [83ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [83ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [161ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [162ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [163ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [163ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@840503851(state=PENDING, result=null, callback=(continuation of Deferred@852402525 after retry RPC after error@214437234) -> (continuation of Deferred@1635202294 after retry RPC after error@453372601) -> (continuation of Deferred@1543922893 after retry RPC after error@283722146) -> (continuation of Deferred@1214318541 after retry RPC after error@2080730202) -> (continuation of Deferred@332029723 after retry RPC after error@1240787089) -> (continuation of Deferred@158056717 after retry RPC after error@1591417798) -> (continuation of Deferred@531150209 after retry RPC after error@1903588997) -> (continuation of Deferred@641901071 after retry RPC after error@541169706), errback=(continuation of Deferred@852402525 after retry RPC after error@214437234) -> (continuation of Deferred@1635202294 after retry RPC after error@453372601) -> (continuation of Deferred@1543922893 after retry RPC after error@283722146) -> (continuation of Deferred@1214318541 after retry RPC after error@2080730202) -> (continuation of Deferred@332029723 after retry RPC after error@1240787089) -> (continuation of Deferred@158056717 after retry RPC after error@1591417798) -> (continuation of Deferred@531150209 after retry RPC after error@1903588997) -> (continuation of Deferred@641901071 after retry RPC after error@541169706)))
20:27:52.051 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:52.051 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:52.052 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:52.052 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:52.052 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:52.052 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:52.052 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:52.052 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:52.052 [WARN - kudu-nio-7] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:52.071 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:52.071 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:52.072 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:52.072 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:52.072 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:52.072 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:52.072 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:52.072 [INFO - pool-40-thread-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:52.072 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:52.091 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:52.091 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:52.091 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:52.092 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:52.092 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:52.092 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:52.092 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:52.092 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:52.093 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:52.111 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:52.112 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:52.112 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:52.112 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:52.112 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:52.112 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:52.112 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:52.113 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:52.114 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:52.131 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:52.131 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:52.131 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:52.131 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:52.131 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:52.132 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:52.132 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:52.132 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:52.133 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:52.151 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:52.151 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:52.152 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:52.152 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:52.152 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:52.153 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:52.153 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:52.153 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:52.153 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:52.191 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:52.191 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:52.192 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:52.192 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:52.192 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:52.192 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:52.192 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:52.192 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:52.193 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:52.231 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:52.232 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:52.233 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:52.233 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:52.233 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:52.233 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:52.233 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:52.233 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:52.233 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:52.234 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [43ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [62ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [63ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [64ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [64ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [83ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [103ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [103ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [141ms] refreshing cache from master, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [143ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [181ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [182ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [183ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [183ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [183ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@1636347888(state=PENDING, result=null, callback=(continuation of Deferred@457588392 after retry RPC after error@799469957) -> (continuation of Deferred@1063769067 after retry RPC after error@464784600) -> (continuation of Deferred@825082488 after retry RPC after error@698300161) -> (continuation of Deferred@677794130 after retry RPC after error@319773258) -> (continuation of Deferred@1282303057 after retry RPC after error@471430626) -> (continuation of Deferred@107961330 after retry RPC after error@1876663423) -> (continuation of Deferred@1353195833 after retry RPC after error@130672811), errback=(continuation of Deferred@457588392 after retry RPC after error@799469957) -> (continuation of Deferred@1063769067 after retry RPC after error@464784600) -> (continuation of Deferred@825082488 after retry RPC after error@698300161) -> (continuation of Deferred@677794130 after retry RPC after error@319773258) -> (continuation of Deferred@1282303057 after retry RPC after error@471430626) -> (continuation of Deferred@107961330 after retry RPC after error@1876663423) -> (continuation of Deferred@1353195833 after retry RPC after error@130672811)))
20:27:52.234 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [43ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [62ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [63ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [64ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [64ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [83ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [103ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [103ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [141ms] refreshing cache from master, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [143ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [181ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [182ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [183ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [183ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [183ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@1636347888(state=PENDING, result=null, callback=(continuation of Deferred@457588392 after retry RPC after error@799469957) -> (continuation of Deferred@1063769067 after retry RPC after error@464784600) -> (continuation of Deferred@825082488 after retry RPC after error@698300161) -> (continuation of Deferred@677794130 after retry RPC after error@319773258) -> (continuation of Deferred@1282303057 after retry RPC after error@471430626) -> (continuation of Deferred@107961330 after retry RPC after error@1876663423) -> (continuation of Deferred@1353195833 after retry RPC after error@130672811), errback=(continuation of Deferred@457588392 after retry RPC after error@799469957) -> (continuation of Deferred@1063769067 after retry RPC after error@464784600) -> (continuation of Deferred@825082488 after retry RPC after error@698300161) -> (continuation of Deferred@677794130 after retry RPC after error@319773258) -> (continuation of Deferred@1282303057 after retry RPC after error@471430626) -> (continuation of Deferred@107961330 after retry RPC after error@1876663423) -> (continuation of Deferred@1353195833 after retry RPC after error@130672811)))
20:27:52.251 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:52.251 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:52.252 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:52.252 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:52.252 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:52.252 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:52.252 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:52.252 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:52.252 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:52.271 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:52.272 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:52.272 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:52.272 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:52.272 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:52.272 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:52.272 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:52.272 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:52.273 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:52.290 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:52.291 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:52.291 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:52.291 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:52.291 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:52.291 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:52.291 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:52.291 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:52.292 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:52.310 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:52.311 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:52.311 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:52.311 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:52.311 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:52.311 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:52.312 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:52.312 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:52.312 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:52.331 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:52.331 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:52.331 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:52.331 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:52.331 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:52.331 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:52.332 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:52.332 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:52.333 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:52.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:27:52.334084 3893 master_proxy_rpc.cc:177] Failed to determine new leader Master: Timed out: ConnectToClusterRpc(addrs: 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127, num_attempts: 97) passed its deadline: Not found: no leader found: ConnectToClusterRpc(addrs: 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127, num_attempts: 1)
20:27:52.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:27:52.335582 3893 meta_cache.cc:788] Timed out: LookupRpcById { tablet: 'f9ccfdb32e164e61bde68775baab7f9d', attempt: 2 } failed: LookupRpcById timed out after deadline expired: LookupRpcById { tablet: 'f9ccfdb32e164e61bde68775baab7f9d', attempt: 2 } passed its deadline: Timed out: ConnectToClusterRpc(addrs: 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127, num_attempts: 97) passed its deadline: Not found: no leader found: ConnectToClusterRpc(addrs: 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127, num_attempts: 1)
20:27:52.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:27:52.337704 3893 master_proxy_rpc.cc:203] Re-attempting LookupRpcById request to leader Master (127.3.46.62:46071)
20:27:52.371 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:52.371 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:52.372 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:52.372 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:52.372 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:52.372 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:52.372 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:52.372 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:52.372 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:52.411 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:52.411 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:52.412 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:52.412 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:52.412 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:52.412 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:52.412 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:52.412 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:52.412 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:52.451 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:52.452 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:52.452 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:52.452 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:52.452 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:52.452 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:52.452 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:52.452 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:52.452 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:52.453 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=201), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [1ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [40ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [60ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [80ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [119ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [121ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [161ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@1428066604(state=PENDING, result=null, callback=(continuation of Deferred@286759100 after retry RPC after error@1290949517) -> (continuation of Deferred@699393520 after retry RPC after error@135890456) -> (continuation of Deferred@627453407 after retry RPC after error@2077650117) -> (continuation of Deferred@755009066 after retry RPC after error@1777200340) -> (continuation of Deferred@2104569059 after retry RPC after error@889837516) -> (continuation of Deferred@470886033 after retry RPC after error@438104715) -> (continuation of Deferred@1862193951 after retry RPC after error@364039867), errback=(continuation of Deferred@286759100 after retry RPC after error@1290949517) -> (continuation of Deferred@699393520 after retry RPC after error@135890456) -> (continuation of Deferred@627453407 after retry RPC after error@2077650117) -> (continuation of Deferred@755009066 after retry RPC after error@1777200340) -> (continuation of Deferred@2104569059 after retry RPC after error@889837516) -> (continuation of Deferred@470886033 after retry RPC after error@438104715) -> (continuation of Deferred@1862193951 after retry RPC after error@364039867)))
20:27:52.453 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=201), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [1ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [40ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [60ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [80ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [119ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [121ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [161ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@1428066604(state=PENDING, result=null, callback=(continuation of Deferred@286759100 after retry RPC after error@1290949517) -> (continuation of Deferred@699393520 after retry RPC after error@135890456) -> (continuation of Deferred@627453407 after retry RPC after error@2077650117) -> (continuation of Deferred@755009066 after retry RPC after error@1777200340) -> (continuation of Deferred@2104569059 after retry RPC after error@889837516) -> (continuation of Deferred@470886033 after retry RPC after error@438104715) -> (continuation of Deferred@1862193951 after retry RPC after error@364039867), errback=(continuation of Deferred@286759100 after retry RPC after error@1290949517) -> (continuation of Deferred@699393520 after retry RPC after error@135890456) -> (continuation of Deferred@627453407 after retry RPC after error@2077650117) -> (continuation of Deferred@755009066 after retry RPC after error@1777200340) -> (continuation of Deferred@2104569059 after retry RPC after error@889837516) -> (continuation of Deferred@470886033 after retry RPC after error@438104715) -> (continuation of Deferred@1862193951 after retry RPC after error@364039867)))
20:27:52.470 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:52.471 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:52.471 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:52.471 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:52.471 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:52.471 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:52.471 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:52.471 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:52.472 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:52.490 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:52.491 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:52.491 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:52.491 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:52.491 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:52.491 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:52.491 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:52.491 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:52.492 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:52.510 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:52.511 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:52.511 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:52.511 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:52.511 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:52.511 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:52.511 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:52.511 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:52.511 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:52.530 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:52.531 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:52.531 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:52.531 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:52.532 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:52.532 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:52.532 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:52.532 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:52.532 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:52.551 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:52.551 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:52.552 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:52.552 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:52.552 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:52.552 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:52.552 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:52.552 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:52.553 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:52.571 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:52.571 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:52.572 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:52.572 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:52.572 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:52.572 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:52.572 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:52.572 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:52.573 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:52.650 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:52.651 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:52.651 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:52.651 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:52.651 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:52.651 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:52.652 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:52.652 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:52.652 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:52.653 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@1952750249(state=PENDING, result=null, callback=(continuation of Deferred@855455323 after retry RPC after error@509901554) -> (continuation of Deferred@393487097 after retry RPC after error@1200968556) -> (continuation of Deferred@469887531 after retry RPC after error@554676403) -> (continuation of Deferred@391785857 after retry RPC after error@2050556469) -> (continuation of Deferred@1769668587 after retry RPC after error@892092511) -> (continuation of Deferred@1935431006 after retry RPC after error@1095162282), errback=(continuation of Deferred@855455323 after retry RPC after error@509901554) -> (continuation of Deferred@393487097 after retry RPC after error@1200968556) -> (continuation of Deferred@469887531 after retry RPC after error@554676403) -> (continuation of Deferred@391785857 after retry RPC after error@2050556469) -> (continuation of Deferred@1769668587 after retry RPC after error@892092511) -> (continuation of Deferred@1935431006 after retry RPC after error@1095162282)))
20:27:52.653 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@1952750249(state=PENDING, result=null, callback=(continuation of Deferred@855455323 after retry RPC after error@509901554) -> (continuation of Deferred@393487097 after retry RPC after error@1200968556) -> (continuation of Deferred@469887531 after retry RPC after error@554676403) -> (continuation of Deferred@391785857 after retry RPC after error@2050556469) -> (continuation of Deferred@1769668587 after retry RPC after error@892092511) -> (continuation of Deferred@1935431006 after retry RPC after error@1095162282), errback=(continuation of Deferred@855455323 after retry RPC after error@509901554) -> (continuation of Deferred@393487097 after retry RPC after error@1200968556) -> (continuation of Deferred@469887531 after retry RPC after error@554676403) -> (continuation of Deferred@391785857 after retry RPC after error@2050556469) -> (continuation of Deferred@1769668587 after retry RPC after error@892092511) -> (continuation of Deferred@1935431006 after retry RPC after error@1095162282)))
20:27:52.671 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:52.671 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:52.671 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:52.671 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:52.671 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:52.672 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:52.672 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:52.672 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:52.673 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:52.691 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:52.691 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:52.691 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:52.692 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:52.692 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:52.692 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:52.692 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:52.692 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:52.693 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:52.711 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:52.711 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:52.712 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:52.712 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:52.712 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:52.712 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:52.712 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:52.712 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:52.712 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:52.731 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:52.731 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:52.732 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:52.732 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:52.732 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:52.732 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:52.732 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:52.732 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:52.732 [WARN - kudu-nio-3] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:52.750 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:52.751 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:52.751 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:52.751 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:52.751 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:52.751 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:52.752 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:52.752 [INFO - pool-40-thread-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:52.753 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:52.770 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:52.771 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:52.771 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:52.771 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:52.771 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:52.771 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:52.771 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:52.771 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:52.772 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:52.810 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:52.811 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:52.811 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:52.811 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:52.811 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:52.811 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:52.811 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:52.811 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:52.812 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:52.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:52.895049 4319 raft_consensus.cc:493] T 00000000000000000000000000000000 P 0db1d539946942f794b0c4ec81f8f4b4 [term 4 FOLLOWER]: Starting pre-election (detected failure of leader b1731f8308c145f686f88efd6a54b851)
20:27:52.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:52.895257 4319 raft_consensus.cc:515] T 00000000000000000000000000000000 P 0db1d539946942f794b0c4ec81f8f4b4 [term 4 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0db1d539946942f794b0c4ec81f8f4b4" member_type: VOTER last_known_addr { host: "127.3.46.62" port: 46071 } } peers { permanent_uuid: "d6c270b387b04f17b2a58bfcb0eb4f82" member_type: VOTER last_known_addr { host: "127.3.46.61" port: 39407 } } peers { permanent_uuid: "b1731f8308c145f686f88efd6a54b851" member_type: VOTER last_known_addr { host: "127.3.46.60" port: 36127 } }
20:27:52.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:52.895946 4319 leader_election.cc:290] T 00000000000000000000000000000000 P 0db1d539946942f794b0c4ec81f8f4b4 [CANDIDATE]: Term 5 pre-election: Requested pre-vote from peers d6c270b387b04f17b2a58bfcb0eb4f82 (127.3.46.61:39407), b1731f8308c145f686f88efd6a54b851 (127.3.46.60:36127)
20:27:52.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:27:52.898168 4149 leader_election.cc:336] T 00000000000000000000000000000000 P 0db1d539946942f794b0c4ec81f8f4b4 [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer d6c270b387b04f17b2a58bfcb0eb4f82 (127.3.46.61:39407): Network error: Client connection negotiation failed: client connection to 127.3.46.61:39407: connect: Connection refused (error 111)
20:27:52.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:27:52.898384 4148 leader_election.cc:336] T 00000000000000000000000000000000 P 0db1d539946942f794b0c4ec81f8f4b4 [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer b1731f8308c145f686f88efd6a54b851 (127.3.46.60:36127): Network error: Client connection negotiation failed: client connection to 127.3.46.60:36127: connect: Connection refused (error 111)
20:27:52.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:52.898504 4148 leader_election.cc:304] T 00000000000000000000000000000000 P 0db1d539946942f794b0c4ec81f8f4b4 [CANDIDATE]: Term 5 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 0db1d539946942f794b0c4ec81f8f4b4; no voters: b1731f8308c145f686f88efd6a54b851, d6c270b387b04f17b2a58bfcb0eb4f82
20:27:52.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:52.898752 4319 raft_consensus.cc:2749] T 00000000000000000000000000000000 P 0db1d539946942f794b0c4ec81f8f4b4 [term 4 FOLLOWER]: Leader pre-election lost for term 5. Reason: could not achieve majority
20:27:52.931 [DEBUG - pool-40-thread-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=260), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [3ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [23ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [83ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@1637822158(state=PENDING, result=null, callback=(continuation of Deferred@1879955419 after retry RPC after error@2124645465) -> (continuation of Deferred@94571673 after retry RPC after error@500349885) -> (continuation of Deferred@67048659 after retry RPC after error@1629280697) -> (continuation of Deferred@2093405895 after retry RPC after error@1958839729) -> (continuation of Deferred@258170486 after retry RPC after error@137727794) -> (continuation of Deferred@1108283421 after retry RPC after error@983300576) -> (continuation of Deferred@798916651 after retry RPC after error@192333557), errback=(continuation of Deferred@1879955419 after retry RPC after error@2124645465) -> (continuation of Deferred@94571673 after retry RPC after error@500349885) -> (continuation of Deferred@67048659 after retry RPC after error@1629280697) -> (continuation of Deferred@2093405895 after retry RPC after error@1958839729) -> (continuation of Deferred@258170486 after retry RPC after error@137727794) -> (continuation of Deferred@1108283421 after retry RPC after error@983300576) -> (continuation of Deferred@798916651 after retry RPC after error@192333557)))
20:27:52.932 [WARN - pool-40-thread-1] (HashedWheelTimer.java:716) An exception was thrown by TimerTask.
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713)
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701)
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:787)
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501)
at java.base/java.lang.Thread.run(Thread.java:840)
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713)
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701)
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:787)
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501)
at java.base/java.lang.Thread.run(Thread.java:840)
20:27:52.950 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:52.951 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:52.951 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:52.952 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:52.952 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:52.952 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:52.952 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:52.952 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:52.952 [WARN - kudu-nio-3] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:52.971 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:52.971 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:52.971 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:52.972 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:52.972 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:52.972 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:52.972 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:52.972 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:52.972 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:52.991 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:52.991 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:52.992 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:52.992 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:52.992 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:52.992 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:52.992 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:52.992 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:52.992 [WARN - kudu-nio-7] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:53.011 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:53.012 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:53.012 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:53.012 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:53.012 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:53.012 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:53.012 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:53.012 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:53.013 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:53.030 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:53.031 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:53.031 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:53.031 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:53.031 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:53.031 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:53.031 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:53.031 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:53.032 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:53.070 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:53.071 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:53.071 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:53.071 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:53.071 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:53.071 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:53.071 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:53.072 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:53.072 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:53.091 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:53.091 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:53.092 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:53.092 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:53.092 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:53.092 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:53.092 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:53.092 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:53.092 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:53.211 [DEBUG - pool-40-thread-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=260), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [63ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@1199364179(state=PENDING, result=null, callback=(continuation of Deferred@336111267 after retry RPC after error@1434488072) -> (continuation of Deferred@2139502984 after retry RPC after error@1578752107) -> (continuation of Deferred@267731125 after retry RPC after error@1582064782) -> (continuation of Deferred@281566390 after retry RPC after error@136260) -> (continuation of Deferred@626465817 after retry RPC after error@402809897) -> (continuation of Deferred@1258137203 after retry RPC after error@1523395420) -> (continuation of Deferred@1206338504 after retry RPC after error@1288655921), errback=(continuation of Deferred@336111267 after retry RPC after error@1434488072) -> (continuation of Deferred@2139502984 after retry RPC after error@1578752107) -> (continuation of Deferred@267731125 after retry RPC after error@1582064782) -> (continuation of Deferred@281566390 after retry RPC after error@136260) -> (continuation of Deferred@626465817 after retry RPC after error@402809897) -> (continuation of Deferred@1258137203 after retry RPC after error@1523395420) -> (continuation of Deferred@1206338504 after retry RPC after error@1288655921)))
20:27:53.211 [WARN - pool-40-thread-1] (HashedWheelTimer.java:716) An exception was thrown by TimerTask.
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713)
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701)
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:787)
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501)
at java.base/java.lang.Thread.run(Thread.java:840)
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713)
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701)
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:787)
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501)
at java.base/java.lang.Thread.run(Thread.java:840)
20:27:53.231 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:53.231 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:53.232 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:53.232 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:53.232 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:53.232 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:53.232 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:53.232 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:53.232 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:53.251 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:53.252 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:53.252 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:53.252 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:53.252 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:53.252 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:53.252 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:53.252 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:53.254 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:53.270 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:53.271 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:53.271 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:53.271 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:53.271 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:53.271 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:53.272 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:53.272 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:53.272 [WARN - kudu-nio-5] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:53.291 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:53.292 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:53.292 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:53.292 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:53.292 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:53.292 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:53.292 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:53.292 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:53.293 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:53.310 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:53.311 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:53.311 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:53.311 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:53.311 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:53.312 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:53.312 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:53.312 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:53.312 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:53.330 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:53.331 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:53.331 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:53.331 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:53.331 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:53.331 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:53.331 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:53.332 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:53.332 [WARN - kudu-nio-3] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:53.391 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:53.391 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:53.392 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:53.392 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:53.392 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:53.392 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:53.392 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:53.392 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:53.392 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:53.392 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:53.393 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:53.393 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:53.393 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [23ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [24ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [62ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [63ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [162ms] refreshing cache from master, [162ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [162ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [162ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [163ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@1025698287(state=PENDING, result=null, callback=(continuation of Deferred@2059190501 after retry RPC after error@1785785876) -> (continuation of Deferred@751113350 after retry RPC after error@109247840) -> (continuation of Deferred@438681277 after retry RPC after error@2009654179) -> (continuation of Deferred@1932372216 after retry RPC after error@1658135040) -> (continuation of Deferred@1437771953 after retry RPC after error@1700240479) -> (continuation of Deferred@2074313317 after retry RPC after error@816671283), errback=(continuation of Deferred@2059190501 after retry RPC after error@1785785876) -> (continuation of Deferred@751113350 after retry RPC after error@109247840) -> (continuation of Deferred@438681277 after retry RPC after error@2009654179) -> (continuation of Deferred@1932372216 after retry RPC after error@1658135040) -> (continuation of Deferred@1437771953 after retry RPC after error@1700240479) -> (continuation of Deferred@2074313317 after retry RPC after error@816671283)))
20:27:53.394 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [23ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [24ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [62ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [63ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [162ms] refreshing cache from master, [162ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [162ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [162ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [163ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@1025698287(state=PENDING, result=null, callback=(continuation of Deferred@2059190501 after retry RPC after error@1785785876) -> (continuation of Deferred@751113350 after retry RPC after error@109247840) -> (continuation of Deferred@438681277 after retry RPC after error@2009654179) -> (continuation of Deferred@1932372216 after retry RPC after error@1658135040) -> (continuation of Deferred@1437771953 after retry RPC after error@1700240479) -> (continuation of Deferred@2074313317 after retry RPC after error@816671283), errback=(continuation of Deferred@2059190501 after retry RPC after error@1785785876) -> (continuation of Deferred@751113350 after retry RPC after error@109247840) -> (continuation of Deferred@438681277 after retry RPC after error@2009654179) -> (continuation of Deferred@1932372216 after retry RPC after error@1658135040) -> (continuation of Deferred@1437771953 after retry RPC after error@1700240479) -> (continuation of Deferred@2074313317 after retry RPC after error@816671283)))
20:27:53.410 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:53.411 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:53.411 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:53.411 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:53.411 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:53.411 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:53.411 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:53.411 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:53.412 [WARN - kudu-nio-7] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:53.430 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:53.431 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:53.431 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:53.431 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:53.431 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:53.431 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:53.431 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:53.432 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:53.432 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:53.451 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:53.451 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:53.452 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:53.452 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:53.452 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:53.452 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:53.452 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:53.452 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:53.452 [WARN - kudu-nio-3] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:53.471 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:53.471 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:53.472 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:53.472 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:53.472 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:53.472 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:53.472 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:53.472 [INFO - pool-40-thread-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:53.472 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:53.491 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:53.491 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:53.492 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:53.492 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:53.492 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:53.492 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:53.492 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:53.492 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:53.493 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:53.511 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:53.511 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:53.512 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:53.512 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:53.512 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:53.512 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:53.512 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:53.512 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:53.512 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:53.531 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:53.531 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:53.531 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:53.532 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:53.532 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:53.532 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:53.532 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:53.532 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:53.532 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:53.571 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:53.571 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:53.572 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:53.572 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:53.572 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:53.572 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:53.572 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:53.572 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:53.573 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:53.610 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:53.611 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:53.611 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:53.611 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:53.611 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:53.612 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:53.612 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:53.612 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:53.612 [WARN - kudu-nio-7] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:53.613 [DEBUG - kudu-nio-7] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [83ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [100ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [120ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [161ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [202ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@387436185(state=PENDING, result=null, callback=(continuation of Deferred@1408144622 after retry RPC after error@776772625) -> (continuation of Deferred@658424541 after retry RPC after error@250913701) -> (continuation of Deferred@697966449 after retry RPC after error@1677505095) -> (continuation of Deferred@839236815 after retry RPC after error@432523425) -> (continuation of Deferred@1625238911 after retry RPC after error@1714935209) -> (continuation of Deferred@2035023460 after retry RPC after error@1584309036) -> (continuation of Deferred@1204013387 after retry RPC after error@528598663) -> (continuation of Deferred@966767565 after retry RPC after error@314769425), errback=(continuation of Deferred@1408144622 after retry RPC after error@776772625) -> (continuation of Deferred@658424541 after retry RPC after error@250913701) -> (continuation of Deferred@697966449 after retry RPC after error@1677505095) -> (continuation of Deferred@839236815 after retry RPC after error@432523425) -> (continuation of Deferred@1625238911 after retry RPC after error@1714935209) -> (continuation of Deferred@2035023460 after retry RPC after error@1584309036) -> (continuation of Deferred@1204013387 after retry RPC after error@528598663) -> (continuation of Deferred@966767565 after retry RPC after error@314769425)))
20:27:53.613 [DEBUG - kudu-nio-7] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [83ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [100ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [120ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [161ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [202ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@387436185(state=PENDING, result=null, callback=(continuation of Deferred@1408144622 after retry RPC after error@776772625) -> (continuation of Deferred@658424541 after retry RPC after error@250913701) -> (continuation of Deferred@697966449 after retry RPC after error@1677505095) -> (continuation of Deferred@839236815 after retry RPC after error@432523425) -> (continuation of Deferred@1625238911 after retry RPC after error@1714935209) -> (continuation of Deferred@2035023460 after retry RPC after error@1584309036) -> (continuation of Deferred@1204013387 after retry RPC after error@528598663) -> (continuation of Deferred@966767565 after retry RPC after error@314769425), errback=(continuation of Deferred@1408144622 after retry RPC after error@776772625) -> (continuation of Deferred@658424541 after retry RPC after error@250913701) -> (continuation of Deferred@697966449 after retry RPC after error@1677505095) -> (continuation of Deferred@839236815 after retry RPC after error@432523425) -> (continuation of Deferred@1625238911 after retry RPC after error@1714935209) -> (continuation of Deferred@2035023460 after retry RPC after error@1584309036) -> (continuation of Deferred@1204013387 after retry RPC after error@528598663) -> (continuation of Deferred@966767565 after retry RPC after error@314769425)))
20:27:53.630 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:53.631 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:53.631 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:53.631 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:53.631 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:53.631 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:53.631 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:53.632 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:53.632 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:53.650 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:53.651 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:53.651 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:53.651 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:53.651 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:53.651 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:53.651 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:53.651 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:53.652 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:53.670 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:53.671 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:53.671 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:53.671 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:53.671 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:53.671 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:53.671 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:53.671 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:53.672 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:53.690 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:53.691 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:53.691 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:53.691 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:53.691 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:53.691 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:53.692 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:53.692 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:53.692 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:53.711 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:53.711 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:53.711 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:53.711 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:53.711 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:53.711 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:53.711 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:53.712 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:53.712 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:53.731 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:53.731 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:53.732 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:53.732 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:53.732 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:53.732 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:53.732 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:53.732 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:53.732 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:53.751 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:53.751 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:53.752 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:53.752 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:53.752 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:53.752 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:53.752 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:53.752 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:53.752 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:53.771 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:53.772 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:53.772 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:53.772 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:53.772 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:53.772 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:53.772 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:53.772 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:53.773 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:53.773 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=142), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [141ms] refreshing cache from master, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@1492559610(state=PENDING, result=null, callback=(continuation of Deferred@352380134 after retry RPC after error@1649824144) -> (continuation of Deferred@689863813 after retry RPC after error@1152572970) -> (continuation of Deferred@1443313038 after retry RPC after error@109917199) -> (continuation of Deferred@1546492431 after retry RPC after error@238836782) -> (continuation of Deferred@1643688738 after retry RPC after error@247127646) -> (continuation of Deferred@1048035428 after retry RPC after error@1380333224) -> (continuation of Deferred@1988357729 after retry RPC after error@773031617), errback=(continuation of Deferred@352380134 after retry RPC after error@1649824144) -> (continuation of Deferred@689863813 after retry RPC after error@1152572970) -> (continuation of Deferred@1443313038 after retry RPC after error@109917199) -> (continuation of Deferred@1546492431 after retry RPC after error@238836782) -> (continuation of Deferred@1643688738 after retry RPC after error@247127646) -> (continuation of Deferred@1048035428 after retry RPC after error@1380333224) -> (continuation of Deferred@1988357729 after retry RPC after error@773031617)))
20:27:53.773 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=142), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [141ms] refreshing cache from master, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@1492559610(state=PENDING, result=null, callback=(continuation of Deferred@352380134 after retry RPC after error@1649824144) -> (continuation of Deferred@689863813 after retry RPC after error@1152572970) -> (continuation of Deferred@1443313038 after retry RPC after error@109917199) -> (continuation of Deferred@1546492431 after retry RPC after error@238836782) -> (continuation of Deferred@1643688738 after retry RPC after error@247127646) -> (continuation of Deferred@1048035428 after retry RPC after error@1380333224) -> (continuation of Deferred@1988357729 after retry RPC after error@773031617), errback=(continuation of Deferred@352380134 after retry RPC after error@1649824144) -> (continuation of Deferred@689863813 after retry RPC after error@1152572970) -> (continuation of Deferred@1443313038 after retry RPC after error@109917199) -> (continuation of Deferred@1546492431 after retry RPC after error@238836782) -> (continuation of Deferred@1643688738 after retry RPC after error@247127646) -> (continuation of Deferred@1048035428 after retry RPC after error@1380333224) -> (continuation of Deferred@1988357729 after retry RPC after error@773031617)))
20:27:53.790 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:53.791 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:53.791 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:53.791 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:53.791 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:53.791 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:53.792 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:53.792 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:53.792 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:53.811 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:53.811 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:53.811 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:53.812 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:53.812 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:53.812 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:53.812 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:53.812 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:53.813 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:53.831 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:53.831 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:53.831 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:53.831 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:53.832 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:53.832 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:53.832 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:53.832 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:53.833 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:53.850 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:53.851 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:53.851 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:53.851 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:53.851 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:53.851 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:53.851 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:53.851 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:53.852 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:53.870 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:53.871 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:53.871 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:53.871 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:53.871 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:53.871 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:53.871 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:53.871 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:53.872 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:53.890 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:53.891 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:53.891 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:53.891 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:53.891 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:53.891 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:53.891 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:53.891 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:53.892 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:53.910 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:53.911 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:53.911 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:53.911 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:53.911 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:53.912 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:53.912 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:53.912 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:53.912 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:54.030 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:54.031 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:54.031 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:54.031 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:54.031 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:54.031 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:54.031 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:54.032 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:54.032 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:54.032 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [23ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [43ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@1896789301(state=PENDING, result=null, callback=(continuation of Deferred@569764032 after retry RPC after error@2120804679) -> (continuation of Deferred@1492308746 after retry RPC after error@1557226044) -> (continuation of Deferred@1391207054 after retry RPC after error@1811405637) -> (continuation of Deferred@626212786 after retry RPC after error@1925474726) -> (continuation of Deferred@1091558690 after retry RPC after error@351738805) -> (continuation of Deferred@677713974 after retry RPC after error@2034093509) -> (continuation of Deferred@1181022477 after retry RPC after error@1793222855), errback=(continuation of Deferred@569764032 after retry RPC after error@2120804679) -> (continuation of Deferred@1492308746 after retry RPC after error@1557226044) -> (continuation of Deferred@1391207054 after retry RPC after error@1811405637) -> (continuation of Deferred@626212786 after retry RPC after error@1925474726) -> (continuation of Deferred@1091558690 after retry RPC after error@351738805) -> (continuation of Deferred@677713974 after retry RPC after error@2034093509) -> (continuation of Deferred@1181022477 after retry RPC after error@1793222855)))
20:27:54.033 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [23ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [43ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@1896789301(state=PENDING, result=null, callback=(continuation of Deferred@569764032 after retry RPC after error@2120804679) -> (continuation of Deferred@1492308746 after retry RPC after error@1557226044) -> (continuation of Deferred@1391207054 after retry RPC after error@1811405637) -> (continuation of Deferred@626212786 after retry RPC after error@1925474726) -> (continuation of Deferred@1091558690 after retry RPC after error@351738805) -> (continuation of Deferred@677713974 after retry RPC after error@2034093509) -> (continuation of Deferred@1181022477 after retry RPC after error@1793222855), errback=(continuation of Deferred@569764032 after retry RPC after error@2120804679) -> (continuation of Deferred@1492308746 after retry RPC after error@1557226044) -> (continuation of Deferred@1391207054 after retry RPC after error@1811405637) -> (continuation of Deferred@626212786 after retry RPC after error@1925474726) -> (continuation of Deferred@1091558690 after retry RPC after error@351738805) -> (continuation of Deferred@677713974 after retry RPC after error@2034093509) -> (continuation of Deferred@1181022477 after retry RPC after error@1793222855)))
20:27:54.050 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:54.051 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:54.051 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:54.051 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:54.051 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:54.051 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:54.051 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:54.052 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:54.052 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:54.071 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:54.071 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:54.071 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:54.071 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:54.072 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:54.072 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:54.072 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:54.072 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:54.072 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:54.091 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:54.091 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:54.091 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:54.091 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:54.092 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:54.092 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:54.092 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:54.092 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:54.092 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:54.111 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:54.111 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:54.112 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:54.112 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:54.112 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:54.112 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:54.112 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:54.112 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:54.113 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:54.131 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:54.131 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:54.131 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:54.132 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:54.132 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:54.132 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:54.132 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:54.132 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:54.132 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:54.151 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:54.151 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:54.152 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:54.152 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:54.152 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:54.152 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:54.152 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:54.152 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:54.153 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:54.210 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:54.211 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:54.211 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:54.211 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:54.211 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:54.211 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:54.211 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:54.211 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:54.212 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:54.271 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:54.271 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:54.271 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:54.271 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:54.272 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:54.272 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:54.272 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:54.272 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:54.272 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:54.273 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [63ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [222ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [222ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [222ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@873580328(state=PENDING, result=null, callback=(continuation of Deferred@1103894042 after retry RPC after error@1405022870) -> (continuation of Deferred@2070861919 after retry RPC after error@1098676381) -> (continuation of Deferred@1612827246 after retry RPC after error@1143975614) -> (continuation of Deferred@1234053700 after retry RPC after error@1804864376) -> (continuation of Deferred@92856030 after retry RPC after error@1022085775) -> (continuation of Deferred@68094812 after retry RPC after error@1334517614) -> (continuation of Deferred@204887663 after retry RPC after error@1852529364), errback=(continuation of Deferred@1103894042 after retry RPC after error@1405022870) -> (continuation of Deferred@2070861919 after retry RPC after error@1098676381) -> (continuation of Deferred@1612827246 after retry RPC after error@1143975614) -> (continuation of Deferred@1234053700 after retry RPC after error@1804864376) -> (continuation of Deferred@92856030 after retry RPC after error@1022085775) -> (continuation of Deferred@68094812 after retry RPC after error@1334517614) -> (continuation of Deferred@204887663 after retry RPC after error@1852529364)))
20:27:54.273 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [63ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [222ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [222ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [222ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@873580328(state=PENDING, result=null, callback=(continuation of Deferred@1103894042 after retry RPC after error@1405022870) -> (continuation of Deferred@2070861919 after retry RPC after error@1098676381) -> (continuation of Deferred@1612827246 after retry RPC after error@1143975614) -> (continuation of Deferred@1234053700 after retry RPC after error@1804864376) -> (continuation of Deferred@92856030 after retry RPC after error@1022085775) -> (continuation of Deferred@68094812 after retry RPC after error@1334517614) -> (continuation of Deferred@204887663 after retry RPC after error@1852529364), errback=(continuation of Deferred@1103894042 after retry RPC after error@1405022870) -> (continuation of Deferred@2070861919 after retry RPC after error@1098676381) -> (continuation of Deferred@1612827246 after retry RPC after error@1143975614) -> (continuation of Deferred@1234053700 after retry RPC after error@1804864376) -> (continuation of Deferred@92856030 after retry RPC after error@1022085775) -> (continuation of Deferred@68094812 after retry RPC after error@1334517614) -> (continuation of Deferred@204887663 after retry RPC after error@1852529364)))
20:27:54.291 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:54.291 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:54.291 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:54.292 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:54.292 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:54.292 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:54.292 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:54.292 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:54.292 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:54.311 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:54.311 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:54.312 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:54.312 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:54.312 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:54.312 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:54.312 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:54.312 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:54.312 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:54.331 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:54.331 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:54.332 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:54.332 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:54.332 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:54.332 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:54.332 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:54.332 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:54.333 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:54.351 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:54.352 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:54.352 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:54.352 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:54.352 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:54.352 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:54.352 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:54.352 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:54.353 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:54.370 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:54.371 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:54.371 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:54.371 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:54.371 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:54.371 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:54.371 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:54.371 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:54.372 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:54.411 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:54.411 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:54.411 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:54.411 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:54.411 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:54.412 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:54.412 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:54.412 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:54.412 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:54.491 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:54.491 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:54.492 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:54.492 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:54.492 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:54.492 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:54.492 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:54.492 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:54.493 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:54.493 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [62ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [63ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [201ms] refreshing cache from master, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [203ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [203ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@169707674(state=PENDING, result=null, callback=(continuation of Deferred@548783986 after retry RPC after error@1771760313) -> (continuation of Deferred@1581317426 after retry RPC after error@947906360) -> (continuation of Deferred@914303250 after retry RPC after error@143573574) -> (continuation of Deferred@1375602440 after retry RPC after error@385842216) -> (continuation of Deferred@1493688412 after retry RPC after error@669353416) -> (continuation of Deferred@100794491 after retry RPC after error@1802576559), errback=(continuation of Deferred@548783986 after retry RPC after error@1771760313) -> (continuation of Deferred@1581317426 after retry RPC after error@947906360) -> (continuation of Deferred@914303250 after retry RPC after error@143573574) -> (continuation of Deferred@1375602440 after retry RPC after error@385842216) -> (continuation of Deferred@1493688412 after retry RPC after error@669353416) -> (continuation of Deferred@100794491 after retry RPC after error@1802576559)))
20:27:54.493 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [62ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [63ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [201ms] refreshing cache from master, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [203ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [203ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@169707674(state=PENDING, result=null, callback=(continuation of Deferred@548783986 after retry RPC after error@1771760313) -> (continuation of Deferred@1581317426 after retry RPC after error@947906360) -> (continuation of Deferred@914303250 after retry RPC after error@143573574) -> (continuation of Deferred@1375602440 after retry RPC after error@385842216) -> (continuation of Deferred@1493688412 after retry RPC after error@669353416) -> (continuation of Deferred@100794491 after retry RPC after error@1802576559), errback=(continuation of Deferred@548783986 after retry RPC after error@1771760313) -> (continuation of Deferred@1581317426 after retry RPC after error@947906360) -> (continuation of Deferred@914303250 after retry RPC after error@143573574) -> (continuation of Deferred@1375602440 after retry RPC after error@385842216) -> (continuation of Deferred@1493688412 after retry RPC after error@669353416) -> (continuation of Deferred@100794491 after retry RPC after error@1802576559)))
20:27:54.510 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:54.511 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:54.511 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:54.511 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:54.511 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:54.511 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:54.511 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:54.511 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:54.512 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:54.530 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:54.531 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:54.531 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:54.531 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:54.531 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:54.531 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:54.531 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:54.531 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:54.532 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:54.550 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:54.551 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:54.551 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:54.551 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:54.551 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:54.551 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:54.551 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:54.551 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:54.552 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:54.571 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:54.571 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:54.571 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:54.571 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:54.571 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:54.572 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:54.572 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:54.572 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:54.572 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:54.591 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:54.591 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:54.592 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:54.592 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:54.592 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:54.592 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:54.592 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:54.592 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:54.592 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:54.611 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:54.611 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:54.612 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:54.612 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:54.612 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:54.612 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:54.612 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:54.612 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:54.612 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:54.670 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:54.671 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:54.672 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:54.672 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:54.671 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:54.672 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:54.673 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:54.673 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:54.673 [WARN - kudu-nio-3] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:54.673 [DEBUG - kudu-nio-3] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [163ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [163ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@381778246(state=PENDING, result=null, callback=(continuation of Deferred@1360322498 after retry RPC after error@1028161524) -> (continuation of Deferred@1650338134 after retry RPC after error@2143844114) -> (continuation of Deferred@139812326 after retry RPC after error@1989948615) -> (continuation of Deferred@1704703963 after retry RPC after error@1507273885) -> (continuation of Deferred@2077225004 after retry RPC after error@1818417566) -> (continuation of Deferred@1426038438 after retry RPC after error@883749728), errback=(continuation of Deferred@1360322498 after retry RPC after error@1028161524) -> (continuation of Deferred@1650338134 after retry RPC after error@2143844114) -> (continuation of Deferred@139812326 after retry RPC after error@1989948615) -> (continuation of Deferred@1704703963 after retry RPC after error@1507273885) -> (continuation of Deferred@2077225004 after retry RPC after error@1818417566) -> (continuation of Deferred@1426038438 after retry RPC after error@883749728)))
20:27:54.674 [DEBUG - kudu-nio-3] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [163ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [163ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@381778246(state=PENDING, result=null, callback=(continuation of Deferred@1360322498 after retry RPC after error@1028161524) -> (continuation of Deferred@1650338134 after retry RPC after error@2143844114) -> (continuation of Deferred@139812326 after retry RPC after error@1989948615) -> (continuation of Deferred@1704703963 after retry RPC after error@1507273885) -> (continuation of Deferred@2077225004 after retry RPC after error@1818417566) -> (continuation of Deferred@1426038438 after retry RPC after error@883749728), errback=(continuation of Deferred@1360322498 after retry RPC after error@1028161524) -> (continuation of Deferred@1650338134 after retry RPC after error@2143844114) -> (continuation of Deferred@139812326 after retry RPC after error@1989948615) -> (continuation of Deferred@1704703963 after retry RPC after error@1507273885) -> (continuation of Deferred@2077225004 after retry RPC after error@1818417566) -> (continuation of Deferred@1426038438 after retry RPC after error@883749728)))
20:27:54.691 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:54.691 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:54.691 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:54.691 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:54.691 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:54.692 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:54.692 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:54.692 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:54.692 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:54.711 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:54.711 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:54.711 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:54.711 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:54.712 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:54.712 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:54.712 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:54.712 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:54.712 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:54.731 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:54.731 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:54.732 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:54.732 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:54.732 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:54.732 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:54.732 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:54.732 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:54.732 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:54.751 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:54.751 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:54.752 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:54.752 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:54.752 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:54.752 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:54.752 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:54.752 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:54.752 [WARN - kudu-nio-3] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:54.771 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:54.772 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:54.772 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:54.772 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:54.772 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:54.772 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:54.772 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:54.772 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:54.773 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:54.790 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:54.791 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:54.791 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:54.791 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:54.791 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:54.791 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:54.791 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:54.791 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:54.791 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:54.791 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:54.792 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:54.793 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:54.811 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:54.811 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:54.811 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:54.811 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:54.811 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:54.811 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:54.811 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:54.811 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:54.812 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:54.871 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:54.871 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:54.872 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:54.872 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:54.872 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:54.872 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:54.872 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:54.872 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:54.872 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:54.911 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:54.911 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:54.912 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:54.912 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:54.912 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:54.912 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:54.912 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:54.912 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:54.913 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:54.913 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [83ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [181ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [221ms] refreshing cache from master, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [222ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [222ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [222ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [223ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@246395028(state=PENDING, result=null, callback=(continuation of Deferred@1606724245 after retry RPC after error@1974095996) -> (continuation of Deferred@1721997221 after retry RPC after error@360423669) -> (continuation of Deferred@1863393485 after retry RPC after error@1210037475) -> (continuation of Deferred@846929146 after retry RPC after error@2052025928) -> (continuation of Deferred@482229892 after retry RPC after error@699611484) -> (continuation of Deferred@1137011804 after retry RPC after error@1123641154) -> (continuation of Deferred@803089287 after retry RPC after error@521641721) -> (continuation of Deferred@2002302667 after retry RPC after error@2123504661), errback=(continuation of Deferred@1606724245 after retry RPC after error@1974095996) -> (continuation of Deferred@1721997221 after retry RPC after error@360423669) -> (continuation of Deferred@1863393485 after retry RPC after error@1210037475) -> (continuation of Deferred@846929146 after retry RPC after error@2052025928) -> (continuation of Deferred@482229892 after retry RPC after error@699611484) -> (continuation of Deferred@1137011804 after retry RPC after error@1123641154) -> (continuation of Deferred@803089287 after retry RPC after error@521641721) -> (continuation of Deferred@2002302667 after retry RPC after error@2123504661)))
20:27:54.913 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [83ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [181ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [221ms] refreshing cache from master, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [222ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [222ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [222ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [223ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@246395028(state=PENDING, result=null, callback=(continuation of Deferred@1606724245 after retry RPC after error@1974095996) -> (continuation of Deferred@1721997221 after retry RPC after error@360423669) -> (continuation of Deferred@1863393485 after retry RPC after error@1210037475) -> (continuation of Deferred@846929146 after retry RPC after error@2052025928) -> (continuation of Deferred@482229892 after retry RPC after error@699611484) -> (continuation of Deferred@1137011804 after retry RPC after error@1123641154) -> (continuation of Deferred@803089287 after retry RPC after error@521641721) -> (continuation of Deferred@2002302667 after retry RPC after error@2123504661), errback=(continuation of Deferred@1606724245 after retry RPC after error@1974095996) -> (continuation of Deferred@1721997221 after retry RPC after error@360423669) -> (continuation of Deferred@1863393485 after retry RPC after error@1210037475) -> (continuation of Deferred@846929146 after retry RPC after error@2052025928) -> (continuation of Deferred@482229892 after retry RPC after error@699611484) -> (continuation of Deferred@1137011804 after retry RPC after error@1123641154) -> (continuation of Deferred@803089287 after retry RPC after error@521641721) -> (continuation of Deferred@2002302667 after retry RPC after error@2123504661)))
20:27:54.931 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:54.931 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:54.932 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:54.932 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:54.932 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:54.932 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:54.932 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:54.932 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:54.933 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:54.950 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:54.951 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:54.951 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:54.951 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:54.951 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:54.952 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:54.952 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:54.952 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:54.952 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:54.971 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:54.971 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:54.972 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:54.972 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:54.972 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:54.972 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:54.972 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:54.972 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:54.972 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:54.991 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:54.991 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:54.992 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:54.992 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:54.992 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:54.992 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:54.992 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:54.992 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:54.992 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:55.011 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:55.011 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:55.012 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:55.012 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:55.012 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:55.012 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:55.012 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:55.012 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:55.013 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:55.031 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:55.031 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:55.032 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:55.032 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:55.032 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:55.033 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:55.033 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:55.033 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:55.033 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:55.070 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:55.071 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:55.071 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:55.071 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:55.071 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:55.071 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:55.071 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:55.071 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:55.072 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:55.090 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:55.091 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:55.091 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:55.091 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:55.091 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:55.091 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:55.092 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:55.092 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:55.092 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:55.093 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=161), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [20ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [39ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [139ms] refreshing cache from master, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [140ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [140ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [141ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [159ms] refreshing cache from master, [159ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [159ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [160ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [161ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@197471626(state=PENDING, result=null, callback=(continuation of Deferred@1629826719 after retry RPC after error@1506401970) -> (continuation of Deferred@1773484289 after retry RPC after error@1270800576) -> (continuation of Deferred@52920062 after retry RPC after error@336333926) -> (continuation of Deferred@1420903127 after retry RPC after error@1748303969) -> (continuation of Deferred@1356677911 after retry RPC after error@616958550) -> (continuation of Deferred@2050966499 after retry RPC after error@977887949) -> (continuation of Deferred@11946856 after retry RPC after error@347501970), errback=(continuation of Deferred@1629826719 after retry RPC after error@1506401970) -> (continuation of Deferred@1773484289 after retry RPC after error@1270800576) -> (continuation of Deferred@52920062 after retry RPC after error@336333926) -> (continuation of Deferred@1420903127 after retry RPC after error@1748303969) -> (continuation of Deferred@1356677911 after retry RPC after error@616958550) -> (continuation of Deferred@2050966499 after retry RPC after error@977887949) -> (continuation of Deferred@11946856 after retry RPC after error@347501970)))
20:27:55.093 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=161), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [20ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [39ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [139ms] refreshing cache from master, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [140ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [140ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [141ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [159ms] refreshing cache from master, [159ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [159ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [160ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [161ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@197471626(state=PENDING, result=null, callback=(continuation of Deferred@1629826719 after retry RPC after error@1506401970) -> (continuation of Deferred@1773484289 after retry RPC after error@1270800576) -> (continuation of Deferred@52920062 after retry RPC after error@336333926) -> (continuation of Deferred@1420903127 after retry RPC after error@1748303969) -> (continuation of Deferred@1356677911 after retry RPC after error@616958550) -> (continuation of Deferred@2050966499 after retry RPC after error@977887949) -> (continuation of Deferred@11946856 after retry RPC after error@347501970), errback=(continuation of Deferred@1629826719 after retry RPC after error@1506401970) -> (continuation of Deferred@1773484289 after retry RPC after error@1270800576) -> (continuation of Deferred@52920062 after retry RPC after error@336333926) -> (continuation of Deferred@1420903127 after retry RPC after error@1748303969) -> (continuation of Deferred@1356677911 after retry RPC after error@616958550) -> (continuation of Deferred@2050966499 after retry RPC after error@977887949) -> (continuation of Deferred@11946856 after retry RPC after error@347501970)))
20:27:55.111 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:55.111 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:55.111 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:55.111 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:55.111 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:55.112 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:55.112 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:55.112 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:55.112 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:55.131 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:55.131 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:55.131 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:55.131 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:55.132 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:55.132 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:55.132 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:55.132 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:55.132 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:55.151 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:55.151 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:55.152 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:55.152 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:55.152 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:55.152 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:55.152 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:55.152 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:55.152 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:55.171 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:55.171 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:55.172 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:55.172 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:55.172 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:55.172 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:55.172 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:55.172 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:55.173 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:55.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:55.189587 4132 master_runner.cc:186] Error getting master registration for 127.3.46.61:39407: Network error: Client connection negotiation failed: client connection to 127.3.46.61:39407: connect: Connection refused (error 111),
20:27:55.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:55.189831 4132 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
20:27:55.190 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:55.190 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:55.191 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:55.191 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:55.191 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:55.191 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:55.191 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:55.191 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:55.191 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:55.211 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:55.211 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:55.212 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:55.212 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:55.212 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:55.212 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:55.212 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:55.212 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:55.212 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:55.231 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:55.231 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:55.232 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:55.232 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:55.232 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:55.232 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:55.232 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:55.232 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:55.233 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:55.290 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:55.291 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:55.291 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:55.291 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:55.291 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:55.291 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:55.291 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:55.291 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:55.292 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:55.292 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=181), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [63ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [123ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@1410160125(state=PENDING, result=null, callback=(continuation of Deferred@1942862726 after retry RPC after error@432636443) -> (continuation of Deferred@991202032 after retry RPC after error@84443858) -> (continuation of Deferred@461838462 after retry RPC after error@1309378639) -> (continuation of Deferred@1364711540 after retry RPC after error@269012638) -> (continuation of Deferred@871329801 after retry RPC after error@557109195) -> (continuation of Deferred@1322203105 after retry RPC after error@409300120) -> (continuation of Deferred@1346517157 after retry RPC after error@1460662257), errback=(continuation of Deferred@1942862726 after retry RPC after error@432636443) -> (continuation of Deferred@991202032 after retry RPC after error@84443858) -> (continuation of Deferred@461838462 after retry RPC after error@1309378639) -> (continuation of Deferred@1364711540 after retry RPC after error@269012638) -> (continuation of Deferred@871329801 after retry RPC after error@557109195) -> (continuation of Deferred@1322203105 after retry RPC after error@409300120) -> (continuation of Deferred@1346517157 after retry RPC after error@1460662257)))
20:27:55.292 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=181), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [63ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [123ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@1410160125(state=PENDING, result=null, callback=(continuation of Deferred@1942862726 after retry RPC after error@432636443) -> (continuation of Deferred@991202032 after retry RPC after error@84443858) -> (continuation of Deferred@461838462 after retry RPC after error@1309378639) -> (continuation of Deferred@1364711540 after retry RPC after error@269012638) -> (continuation of Deferred@871329801 after retry RPC after error@557109195) -> (continuation of Deferred@1322203105 after retry RPC after error@409300120) -> (continuation of Deferred@1346517157 after retry RPC after error@1460662257), errback=(continuation of Deferred@1942862726 after retry RPC after error@432636443) -> (continuation of Deferred@991202032 after retry RPC after error@84443858) -> (continuation of Deferred@461838462 after retry RPC after error@1309378639) -> (continuation of Deferred@1364711540 after retry RPC after error@269012638) -> (continuation of Deferred@871329801 after retry RPC after error@557109195) -> (continuation of Deferred@1322203105 after retry RPC after error@409300120) -> (continuation of Deferred@1346517157 after retry RPC after error@1460662257)))
20:27:55.310 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:55.311 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:55.311 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:55.311 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:55.311 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:55.312 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:55.312 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:55.312 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:55.312 [WARN - kudu-nio-7] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:55.331 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:55.332 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:55.332 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:55.332 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:55.332 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:55.332 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:55.332 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:55.332 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:55.333 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:55.350 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:55.351 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:55.351 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:55.351 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:55.351 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:55.351 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:55.352 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:55.352 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:55.352 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:55.370 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:55.371 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:55.371 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:55.371 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:55.371 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:55.371 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:55.371 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:55.371 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:55.372 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:55.390 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:55.391 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:55.391 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:55.391 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:55.391 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:55.391 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:55.391 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:55.392 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:55.392 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:55.410 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:55.411 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:55.411 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:55.411 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:55.411 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:55.411 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:55.411 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:55.411 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:55.412 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:55.450 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:55.451 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:55.451 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:55.451 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:55.451 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:55.451 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:55.451 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:55.452 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:55.452 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:55.491 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:55.491 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:55.492 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:55.492 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:55.492 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:55.492 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:55.492 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:55.492 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:55.492 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:55.493 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [22ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [23ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, deferred=Deferred@1786951123(state=PENDING, result=null, callback=(continuation of Deferred@1961300008 after retry RPC after error@358388948) -> (continuation of Deferred@1188790121 after retry RPC after error@965240396) -> (continuation of Deferred@1965180677 after retry RPC after error@69830465) -> (continuation of Deferred@420774042 after retry RPC after error@1666461682) -> (continuation of Deferred@69818098 after retry RPC after error@1290980025) -> (continuation of Deferred@1910477805 after retry RPC after error@859183515) -> (continuation of Deferred@2112243657 after retry RPC after error@656141774), errback=(continuation of Deferred@1961300008 after retry RPC after error@358388948) -> (continuation of Deferred@1188790121 after retry RPC after error@965240396) -> (continuation of Deferred@1965180677 after retry RPC after error@69830465) -> (continuation of Deferred@420774042 after retry RPC after error@1666461682) -> (continuation of Deferred@69818098 after retry RPC after error@1290980025) -> (continuation of Deferred@1910477805 after retry RPC after error@859183515) -> (continuation of Deferred@2112243657 after retry RPC after error@656141774)))
20:27:55.493 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [22ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [23ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, deferred=Deferred@1786951123(state=PENDING, result=null, callback=(continuation of Deferred@1961300008 after retry RPC after error@358388948) -> (continuation of Deferred@1188790121 after retry RPC after error@965240396) -> (continuation of Deferred@1965180677 after retry RPC after error@69830465) -> (continuation of Deferred@420774042 after retry RPC after error@1666461682) -> (continuation of Deferred@69818098 after retry RPC after error@1290980025) -> (continuation of Deferred@1910477805 after retry RPC after error@859183515) -> (continuation of Deferred@2112243657 after retry RPC after error@656141774), errback=(continuation of Deferred@1961300008 after retry RPC after error@358388948) -> (continuation of Deferred@1188790121 after retry RPC after error@965240396) -> (continuation of Deferred@1965180677 after retry RPC after error@69830465) -> (continuation of Deferred@420774042 after retry RPC after error@1666461682) -> (continuation of Deferred@69818098 after retry RPC after error@1290980025) -> (continuation of Deferred@1910477805 after retry RPC after error@859183515) -> (continuation of Deferred@2112243657 after retry RPC after error@656141774)))
20:27:55.511 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:55.511 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:55.512 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:55.512 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:55.512 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:55.512 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:55.512 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:55.512 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:55.512 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:55.531 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:55.531 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:55.532 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:55.532 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:55.532 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:55.532 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:55.532 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:55.532 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:55.533 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:55.551 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:55.551 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:55.551 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:55.552 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:55.552 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:55.552 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:55.552 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:55.552 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:55.553 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:55.571 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:55.571 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:55.572 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:55.572 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:55.572 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:55.572 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:55.572 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:55.572 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:55.572 [WARN - kudu-nio-5] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:55.591 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:55.591 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:55.591 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:55.591 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:55.592 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:55.592 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:55.592 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:55.592 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:55.592 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:55.611 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:55.611 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:55.611 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:55.612 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:55.612 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:55.612 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:55.612 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:55.612 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:55.612 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:55.631 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:55.631 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:55.632 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:55.632 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:55.632 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:55.632 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:55.632 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:55.632 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:55.632 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:55.671 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:55.671 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:55.672 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:55.672 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:55.672 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:55.672 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:55.672 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:55.672 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:55.673 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:55.673 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [23ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [60ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [100ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [120ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [161ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [163ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@107615291(state=PENDING, result=null, callback=(continuation of Deferred@1425440489 after retry RPC after error@1998279761) -> (continuation of Deferred@1128826820 after retry RPC after error@357931098) -> (continuation of Deferred@1924377903 after retry RPC after error@1604835060) -> (continuation of Deferred@1482034275 after retry RPC after error@1635305595) -> (continuation of Deferred@248368046 after retry RPC after error@1874636021) -> (continuation of Deferred@132218997 after retry RPC after error@545106914) -> (continuation of Deferred@1083020536 after retry RPC after error@1255953176), errback=(continuation of Deferred@1425440489 after retry RPC after error@1998279761) -> (continuation of Deferred@1128826820 after retry RPC after error@357931098) -> (continuation of Deferred@1924377903 after retry RPC after error@1604835060) -> (continuation of Deferred@1482034275 after retry RPC after error@1635305595) -> (continuation of Deferred@248368046 after retry RPC after error@1874636021) -> (continuation of Deferred@132218997 after retry RPC after error@545106914) -> (continuation of Deferred@1083020536 after retry RPC after error@1255953176)))
20:27:55.673 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [23ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [60ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [100ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [120ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [161ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [163ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@107615291(state=PENDING, result=null, callback=(continuation of Deferred@1425440489 after retry RPC after error@1998279761) -> (continuation of Deferred@1128826820 after retry RPC after error@357931098) -> (continuation of Deferred@1924377903 after retry RPC after error@1604835060) -> (continuation of Deferred@1482034275 after retry RPC after error@1635305595) -> (continuation of Deferred@248368046 after retry RPC after error@1874636021) -> (continuation of Deferred@132218997 after retry RPC after error@545106914) -> (continuation of Deferred@1083020536 after retry RPC after error@1255953176), errback=(continuation of Deferred@1425440489 after retry RPC after error@1998279761) -> (continuation of Deferred@1128826820 after retry RPC after error@357931098) -> (continuation of Deferred@1924377903 after retry RPC after error@1604835060) -> (continuation of Deferred@1482034275 after retry RPC after error@1635305595) -> (continuation of Deferred@248368046 after retry RPC after error@1874636021) -> (continuation of Deferred@132218997 after retry RPC after error@545106914) -> (continuation of Deferred@1083020536 after retry RPC after error@1255953176)))
20:27:55.690 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:55.691 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:55.691 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:55.691 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:55.691 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:55.691 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:55.691 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:55.691 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:55.692 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:55.710 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:55.711 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:55.711 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:55.711 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:55.711 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:55.711 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:55.711 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:55.711 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:55.712 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:55.730 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:55.731 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:55.731 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:55.731 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:55.731 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:55.731 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:55.731 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:55.731 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:55.732 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:55.750 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:55.751 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:55.751 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:55.751 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:55.751 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:55.751 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:55.751 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:55.751 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:55.752 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:55.770 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:55.771 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:55.771 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:55.771 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:55.771 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:55.771 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:55.771 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:55.771 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:55.772 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:55.790 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:55.791 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:55.791 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:55.791 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:55.791 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:55.791 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:55.791 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:55.791 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:55.792 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:55.810 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:55.811 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:55.811 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:55.811 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:55.811 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:55.811 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:55.811 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:55.811 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:55.812 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:55.931 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:55.932 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:55.932 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:55.932 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:55.932 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:55.932 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:55.932 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:55.932 [INFO - pool-40-thread-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:55.933 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:55.933 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=243), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [241ms] refreshing cache from master, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [243ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [243ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@1701341614(state=PENDING, result=null, callback=(continuation of Deferred@971320683 after retry RPC after error@2055752287) -> (continuation of Deferred@2010879701 after retry RPC after error@1364767897) -> (continuation of Deferred@1103513004 after retry RPC after error@2032412025) -> (continuation of Deferred@1382185398 after retry RPC after error@1175122719) -> (continuation of Deferred@898770263 after retry RPC after error@1841680753) -> (continuation of Deferred@741059254 after retry RPC after error@1026041383) -> (continuation of Deferred@431227976 after retry RPC after error@1513689174), errback=(continuation of Deferred@971320683 after retry RPC after error@2055752287) -> (continuation of Deferred@2010879701 after retry RPC after error@1364767897) -> (continuation of Deferred@1103513004 after retry RPC after error@2032412025) -> (continuation of Deferred@1382185398 after retry RPC after error@1175122719) -> (continuation of Deferred@898770263 after retry RPC after error@1841680753) -> (continuation of Deferred@741059254 after retry RPC after error@1026041383) -> (continuation of Deferred@431227976 after retry RPC after error@1513689174)))
20:27:55.934 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=243), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [241ms] refreshing cache from master, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [243ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [243ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@1701341614(state=PENDING, result=null, callback=(continuation of Deferred@971320683 after retry RPC after error@2055752287) -> (continuation of Deferred@2010879701 after retry RPC after error@1364767897) -> (continuation of Deferred@1103513004 after retry RPC after error@2032412025) -> (continuation of Deferred@1382185398 after retry RPC after error@1175122719) -> (continuation of Deferred@898770263 after retry RPC after error@1841680753) -> (continuation of Deferred@741059254 after retry RPC after error@1026041383) -> (continuation of Deferred@431227976 after retry RPC after error@1513689174), errback=(continuation of Deferred@971320683 after retry RPC after error@2055752287) -> (continuation of Deferred@2010879701 after retry RPC after error@1364767897) -> (continuation of Deferred@1103513004 after retry RPC after error@2032412025) -> (continuation of Deferred@1382185398 after retry RPC after error@1175122719) -> (continuation of Deferred@898770263 after retry RPC after error@1841680753) -> (continuation of Deferred@741059254 after retry RPC after error@1026041383) -> (continuation of Deferred@431227976 after retry RPC after error@1513689174)))
20:27:55.951 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:55.951 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:55.952 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:55.952 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:55.952 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:55.952 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:55.952 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:55.952 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:55.952 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:55.971 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:55.971 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:55.972 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:55.972 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:55.972 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:55.972 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:55.972 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:55.972 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:55.973 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:55.991 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:55.991 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:55.991 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:55.992 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:55.992 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:55.992 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:55.992 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:55.992 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:55.992 [WARN - kudu-nio-3] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:56.010 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:56.011 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:56.011 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:56.011 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:56.011 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:56.011 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:56.011 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:56.011 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:56.012 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:56.030 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:56.031 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:56.031 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:56.031 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:56.031 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:56.031 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:56.031 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:56.031 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:56.032 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:56.070 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:56.071 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:56.071 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:56.071 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:56.071 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:56.071 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:56.072 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:56.072 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:56.072 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:56.131 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:56.131 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:56.131 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:56.132 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:56.132 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:56.132 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:56.132 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:56.132 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:56.132 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:56.132 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=181), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [1ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [60ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [80ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [119ms] refreshing cache from master, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [120ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [120ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [121ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [179ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [181ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@1678172317(state=PENDING, result=null, callback=(continuation of Deferred@678866194 after retry RPC after error@589920653) -> (continuation of Deferred@1924430366 after retry RPC after error@568684253) -> (continuation of Deferred@1021729230 after retry RPC after error@1424734926) -> (continuation of Deferred@98459335 after retry RPC after error@264719143) -> (continuation of Deferred@23157560 after retry RPC after error@139832336) -> (continuation of Deferred@2139808699 after retry RPC after error@1252069662), errback=(continuation of Deferred@678866194 after retry RPC after error@589920653) -> (continuation of Deferred@1924430366 after retry RPC after error@568684253) -> (continuation of Deferred@1021729230 after retry RPC after error@1424734926) -> (continuation of Deferred@98459335 after retry RPC after error@264719143) -> (continuation of Deferred@23157560 after retry RPC after error@139832336) -> (continuation of Deferred@2139808699 after retry RPC after error@1252069662)))
20:27:56.133 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=181), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [1ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [60ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [80ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [119ms] refreshing cache from master, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [120ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [120ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [121ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [179ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [181ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@1678172317(state=PENDING, result=null, callback=(continuation of Deferred@678866194 after retry RPC after error@589920653) -> (continuation of Deferred@1924430366 after retry RPC after error@568684253) -> (continuation of Deferred@1021729230 after retry RPC after error@1424734926) -> (continuation of Deferred@98459335 after retry RPC after error@264719143) -> (continuation of Deferred@23157560 after retry RPC after error@139832336) -> (continuation of Deferred@2139808699 after retry RPC after error@1252069662), errback=(continuation of Deferred@678866194 after retry RPC after error@589920653) -> (continuation of Deferred@1924430366 after retry RPC after error@568684253) -> (continuation of Deferred@1021729230 after retry RPC after error@1424734926) -> (continuation of Deferred@98459335 after retry RPC after error@264719143) -> (continuation of Deferred@23157560 after retry RPC after error@139832336) -> (continuation of Deferred@2139808699 after retry RPC after error@1252069662)))
20:27:56.151 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:56.151 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:56.152 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:56.152 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:56.152 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:56.152 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:56.152 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:56.152 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:56.152 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:56.171 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:56.171 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:56.171 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:56.171 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:56.172 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:56.172 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:56.172 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:56.172 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:56.172 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:56.191 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:56.191 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:56.191 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:56.191 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:56.191 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:56.191 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:56.192 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:56.192 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:56.192 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:56.210 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:56.211 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:56.211 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:56.211 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:56.211 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:56.211 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:56.211 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:56.211 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:56.212 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:56.230 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:56.231 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:56.231 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:56.231 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:56.231 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:56.231 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:56.231 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:56.232 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:56.232 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:56.250 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:56.251 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:56.251 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:56.251 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:56.251 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:56.252 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:56.252 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:56.252 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:56.252 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:56.271 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:56.271 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:56.271 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:56.272 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:56.272 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:56.272 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:56.272 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:56.272 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:56.272 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:56.291 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:56.291 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:56.291 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:56.292 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:56.292 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:56.292 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:56.292 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:56.292 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:56.292 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:56.295 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=141), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [1ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [19ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [40ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [60ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [80ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [99ms] refreshing cache from master, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [101ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [119ms] refreshing cache from master, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [121ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [139ms] refreshing cache from master, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [141ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@1590420687(state=PENDING, result=null, callback=(continuation of Deferred@736868467 after retry RPC after error@186742091) -> (continuation of Deferred@932662659 after retry RPC after error@922434082) -> (continuation of Deferred@1084564654 after retry RPC after error@1085184732) -> (continuation of Deferred@1306016799 after retry RPC after error@580724140) -> (continuation of Deferred@1274340613 after retry RPC after error@2142826205) -> (continuation of Deferred@1938562545 after retry RPC after error@947381865) -> (continuation of Deferred@1414000815 after retry RPC after error@1815159548), errback=(continuation of Deferred@736868467 after retry RPC after error@186742091) -> (continuation of Deferred@932662659 after retry RPC after error@922434082) -> (continuation of Deferred@1084564654 after retry RPC after error@1085184732) -> (continuation of Deferred@1306016799 after retry RPC after error@580724140) -> (continuation of Deferred@1274340613 after retry RPC after error@2142826205) -> (continuation of Deferred@1938562545 after retry RPC after error@947381865) -> (continuation of Deferred@1414000815 after retry RPC after error@1815159548)))
20:27:56.295 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=141), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [1ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [19ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [40ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [60ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [80ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [99ms] refreshing cache from master, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [101ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [119ms] refreshing cache from master, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [121ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [139ms] refreshing cache from master, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [141ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@1590420687(state=PENDING, result=null, callback=(continuation of Deferred@736868467 after retry RPC after error@186742091) -> (continuation of Deferred@932662659 after retry RPC after error@922434082) -> (continuation of Deferred@1084564654 after retry RPC after error@1085184732) -> (continuation of Deferred@1306016799 after retry RPC after error@580724140) -> (continuation of Deferred@1274340613 after retry RPC after error@2142826205) -> (continuation of Deferred@1938562545 after retry RPC after error@947381865) -> (continuation of Deferred@1414000815 after retry RPC after error@1815159548), errback=(continuation of Deferred@736868467 after retry RPC after error@186742091) -> (continuation of Deferred@932662659 after retry RPC after error@922434082) -> (continuation of Deferred@1084564654 after retry RPC after error@1085184732) -> (continuation of Deferred@1306016799 after retry RPC after error@580724140) -> (continuation of Deferred@1274340613 after retry RPC after error@2142826205) -> (continuation of Deferred@1938562545 after retry RPC after error@947381865) -> (continuation of Deferred@1414000815 after retry RPC after error@1815159548)))
20:27:56.311 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:56.311 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:56.312 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:56.312 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:56.312 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:56.312 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:56.312 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:56.312 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:56.313 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:56.331 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:56.331 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:56.332 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:56.332 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:56.332 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:56.332 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:56.332 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:56.332 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:56.332 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:56.351 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:56.351 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:56.351 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:56.351 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:56.352 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:56.352 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:56.352 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:56.352 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:56.352 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:56.371 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:56.371 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:56.371 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:56.371 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:56.372 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:56.372 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:56.372 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:56.372 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:56.372 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:56.391 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:56.391 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:56.392 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:56.392 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:56.392 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:56.392 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:56.392 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:56.392 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:56.392 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:56.411 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:56.411 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:56.411 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:56.412 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:56.412 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:56.412 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:56.412 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:56.412 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:56.412 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:56.490 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:56.491 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:56.491 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:56.491 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:56.491 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:56.491 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:56.491 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:56.491 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:56.492 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:56.492 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=181), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [3ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@649150812(state=PENDING, result=null, callback=(continuation of Deferred@336703520 after retry RPC after error@302659228) -> (continuation of Deferred@723484395 after retry RPC after error@117989749) -> (continuation of Deferred@511791364 after retry RPC after error@1576807070) -> (continuation of Deferred@231010284 after retry RPC after error@1224946920) -> (continuation of Deferred@1570653629 after retry RPC after error@1636445258) -> (continuation of Deferred@1268454424 after retry RPC after error@460316151), errback=(continuation of Deferred@336703520 after retry RPC after error@302659228) -> (continuation of Deferred@723484395 after retry RPC after error@117989749) -> (continuation of Deferred@511791364 after retry RPC after error@1576807070) -> (continuation of Deferred@231010284 after retry RPC after error@1224946920) -> (continuation of Deferred@1570653629 after retry RPC after error@1636445258) -> (continuation of Deferred@1268454424 after retry RPC after error@460316151)))
20:27:56.492 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=181), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [3ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@649150812(state=PENDING, result=null, callback=(continuation of Deferred@336703520 after retry RPC after error@302659228) -> (continuation of Deferred@723484395 after retry RPC after error@117989749) -> (continuation of Deferred@511791364 after retry RPC after error@1576807070) -> (continuation of Deferred@231010284 after retry RPC after error@1224946920) -> (continuation of Deferred@1570653629 after retry RPC after error@1636445258) -> (continuation of Deferred@1268454424 after retry RPC after error@460316151), errback=(continuation of Deferred@336703520 after retry RPC after error@302659228) -> (continuation of Deferred@723484395 after retry RPC after error@117989749) -> (continuation of Deferred@511791364 after retry RPC after error@1576807070) -> (continuation of Deferred@231010284 after retry RPC after error@1224946920) -> (continuation of Deferred@1570653629 after retry RPC after error@1636445258) -> (continuation of Deferred@1268454424 after retry RPC after error@460316151)))
20:27:56.510 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:56.511 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:56.511 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:56.511 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:56.511 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:56.511 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:56.511 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:56.511 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:56.512 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:56.530 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:56.531 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:56.531 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:56.531 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:56.531 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:56.531 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:56.531 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:56.531 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:56.532 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:56.550 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:56.551 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:56.551 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:56.551 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:56.551 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:56.551 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:56.551 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:56.551 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:56.552 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:56.570 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:56.571 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:56.571 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:56.571 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:56.571 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:56.571 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:56.571 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:56.571 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:56.572 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:56.590 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:56.591 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:56.591 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:56.591 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:56.591 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:56.591 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:56.591 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:56.591 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:56.592 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:56.630 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:56.631 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:56.631 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:56.631 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:56.631 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:56.631 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:56.631 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:56.631 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:56.632 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:56.691 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:56.691 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:56.692 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:56.692 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:56.692 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:56.692 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:56.692 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:56.692 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:56.692 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:56.731 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:56.731 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:56.732 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:56.732 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:56.732 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:56.732 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:56.732 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:56.732 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:56.733 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:56.733 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [180ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [221ms] refreshing cache from master, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [222ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [222ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [223ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [223ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@1480303455(state=PENDING, result=null, callback=(continuation of Deferred@106909752 after retry RPC after error@213769581) -> (continuation of Deferred@1280047217 after retry RPC after error@1240435708) -> (continuation of Deferred@894295145 after retry RPC after error@1475946776) -> (continuation of Deferred@1147329510 after retry RPC after error@1902577917) -> (continuation of Deferred@705834028 after retry RPC after error@876442015) -> (continuation of Deferred@1725051157 after retry RPC after error@792456658) -> (continuation of Deferred@351524693 after retry RPC after error@1260256733), errback=(continuation of Deferred@106909752 after retry RPC after error@213769581) -> (continuation of Deferred@1280047217 after retry RPC after error@1240435708) -> (continuation of Deferred@894295145 after retry RPC after error@1475946776) -> (continuation of Deferred@1147329510 after retry RPC after error@1902577917) -> (continuation of Deferred@705834028 after retry RPC after error@876442015) -> (continuation of Deferred@1725051157 after retry RPC after error@792456658) -> (continuation of Deferred@351524693 after retry RPC after error@1260256733)))
20:27:56.733 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [180ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [221ms] refreshing cache from master, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [222ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [222ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [223ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [223ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@1480303455(state=PENDING, result=null, callback=(continuation of Deferred@106909752 after retry RPC after error@213769581) -> (continuation of Deferred@1280047217 after retry RPC after error@1240435708) -> (continuation of Deferred@894295145 after retry RPC after error@1475946776) -> (continuation of Deferred@1147329510 after retry RPC after error@1902577917) -> (continuation of Deferred@705834028 after retry RPC after error@876442015) -> (continuation of Deferred@1725051157 after retry RPC after error@792456658) -> (continuation of Deferred@351524693 after retry RPC after error@1260256733), errback=(continuation of Deferred@106909752 after retry RPC after error@213769581) -> (continuation of Deferred@1280047217 after retry RPC after error@1240435708) -> (continuation of Deferred@894295145 after retry RPC after error@1475946776) -> (continuation of Deferred@1147329510 after retry RPC after error@1902577917) -> (continuation of Deferred@705834028 after retry RPC after error@876442015) -> (continuation of Deferred@1725051157 after retry RPC after error@792456658) -> (continuation of Deferred@351524693 after retry RPC after error@1260256733)))
20:27:56.751 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:56.752 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:56.752 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:56.752 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:56.752 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:56.753 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:56.753 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:56.753 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:56.753 [WARN - kudu-nio-3] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:56.770 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:56.771 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:56.771 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:56.771 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:56.771 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:56.771 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:56.771 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:56.771 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:56.772 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:56.790 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:56.791 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:56.791 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:56.791 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:56.791 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:56.791 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:56.791 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:56.791 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:56.792 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:56.810 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:56.811 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:56.811 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:56.811 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:56.812 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:56.812 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:56.812 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:56.812 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:56.812 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:56.830 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:56.831 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:56.831 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:56.831 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:56.831 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:56.831 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:56.831 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:56.832 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:56.832 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:56.850 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:56.851 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:56.851 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:56.851 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:56.851 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:56.852 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:56.852 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:56.852 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:56.852 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:56.871 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:56.871 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:56.871 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:56.871 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:56.871 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:56.872 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:56.872 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:56.872 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:56.872 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:56.951 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:56.951 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:56.952 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:56.952 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:56.952 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:56.952 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:56.952 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:56.952 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:56.953 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:56.953 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [20ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [40ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [80ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [99ms] refreshing cache from master, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [101ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [119ms] refreshing cache from master, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [120ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [121ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@548938438(state=PENDING, result=null, callback=(continuation of Deferred@1419907966 after retry RPC after error@1217399310) -> (continuation of Deferred@1169203383 after retry RPC after error@620692693) -> (continuation of Deferred@1610799831 after retry RPC after error@671775907) -> (continuation of Deferred@1761596429 after retry RPC after error@888843371) -> (continuation of Deferred@1121455169 after retry RPC after error@961318877) -> (continuation of Deferred@797272901 after retry RPC after error@1316819754) -> (continuation of Deferred@1279636640 after retry RPC after error@288024722), errback=(continuation of Deferred@1419907966 after retry RPC after error@1217399310) -> (continuation of Deferred@1169203383 after retry RPC after error@620692693) -> (continuation of Deferred@1610799831 after retry RPC after error@671775907) -> (continuation of Deferred@1761596429 after retry RPC after error@888843371) -> (continuation of Deferred@1121455169 after retry RPC after error@961318877) -> (continuation of Deferred@797272901 after retry RPC after error@1316819754) -> (continuation of Deferred@1279636640 after retry RPC after error@288024722)))
20:27:56.953 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [20ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [40ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [80ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [99ms] refreshing cache from master, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [101ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [119ms] refreshing cache from master, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [120ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [121ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@548938438(state=PENDING, result=null, callback=(continuation of Deferred@1419907966 after retry RPC after error@1217399310) -> (continuation of Deferred@1169203383 after retry RPC after error@620692693) -> (continuation of Deferred@1610799831 after retry RPC after error@671775907) -> (continuation of Deferred@1761596429 after retry RPC after error@888843371) -> (continuation of Deferred@1121455169 after retry RPC after error@961318877) -> (continuation of Deferred@797272901 after retry RPC after error@1316819754) -> (continuation of Deferred@1279636640 after retry RPC after error@288024722), errback=(continuation of Deferred@1419907966 after retry RPC after error@1217399310) -> (continuation of Deferred@1169203383 after retry RPC after error@620692693) -> (continuation of Deferred@1610799831 after retry RPC after error@671775907) -> (continuation of Deferred@1761596429 after retry RPC after error@888843371) -> (continuation of Deferred@1121455169 after retry RPC after error@961318877) -> (continuation of Deferred@797272901 after retry RPC after error@1316819754) -> (continuation of Deferred@1279636640 after retry RPC after error@288024722)))
20:27:56.971 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:56.971 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:56.972 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:56.972 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:56.972 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:56.972 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:56.972 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:56.972 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:56.973 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:56.991 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:56.991 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:56.992 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:56.992 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:56.992 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:56.992 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:56.992 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:56.992 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:56.993 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:57.011 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:57.011 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:57.012 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:57.012 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:57.012 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:57.012 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:57.012 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:57.012 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:57.013 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:57.031 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:57.032 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:57.032 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:57.032 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:57.032 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:57.032 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:57.032 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:57.032 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:57.032 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:57.032 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:57.033 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:57.034 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:57.050 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:57.051 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:57.051 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:57.051 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:57.051 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:57.051 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:57.051 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:57.051 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:57.052 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:57.091 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:57.091 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:57.091 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:57.091 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:57.092 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:57.092 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:57.092 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:57.092 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:57.092 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:57.111 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:57.111 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:57.111 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:57.111 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:57.111 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:57.112 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:57.112 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:57.112 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:57.112 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:57.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:57.180311 4351 raft_consensus.cc:493] T 00000000000000000000000000000000 P 0db1d539946942f794b0c4ec81f8f4b4 [term 4 FOLLOWER]: Starting pre-election (detected failure of leader b1731f8308c145f686f88efd6a54b851)
20:27:57.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:57.180543 4351 raft_consensus.cc:515] T 00000000000000000000000000000000 P 0db1d539946942f794b0c4ec81f8f4b4 [term 4 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0db1d539946942f794b0c4ec81f8f4b4" member_type: VOTER last_known_addr { host: "127.3.46.62" port: 46071 } } peers { permanent_uuid: "d6c270b387b04f17b2a58bfcb0eb4f82" member_type: VOTER last_known_addr { host: "127.3.46.61" port: 39407 } } peers { permanent_uuid: "b1731f8308c145f686f88efd6a54b851" member_type: VOTER last_known_addr { host: "127.3.46.60" port: 36127 } }
20:27:57.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:57.181382 4351 leader_election.cc:290] T 00000000000000000000000000000000 P 0db1d539946942f794b0c4ec81f8f4b4 [CANDIDATE]: Term 5 pre-election: Requested pre-vote from peers d6c270b387b04f17b2a58bfcb0eb4f82 (127.3.46.61:39407), b1731f8308c145f686f88efd6a54b851 (127.3.46.60:36127)
20:27:57.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:27:57.182574 4149 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.3.46.61:39407: connect: Connection refused (error 111) [suppressed 3 similar messages]
20:27:57.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:27:57.183980 4149 leader_election.cc:336] T 00000000000000000000000000000000 P 0db1d539946942f794b0c4ec81f8f4b4 [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer d6c270b387b04f17b2a58bfcb0eb4f82 (127.3.46.61:39407): Network error: Client connection negotiation failed: client connection to 127.3.46.61:39407: connect: Connection refused (error 111)
20:27:57.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:27:57.184183 4148 leader_election.cc:336] T 00000000000000000000000000000000 P 0db1d539946942f794b0c4ec81f8f4b4 [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer b1731f8308c145f686f88efd6a54b851 (127.3.46.60:36127): Network error: Client connection negotiation failed: client connection to 127.3.46.60:36127: connect: Connection refused (error 111)
20:27:57.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:57.184365 4148 leader_election.cc:304] T 00000000000000000000000000000000 P 0db1d539946942f794b0c4ec81f8f4b4 [CANDIDATE]: Term 5 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 0db1d539946942f794b0c4ec81f8f4b4; no voters: b1731f8308c145f686f88efd6a54b851, d6c270b387b04f17b2a58bfcb0eb4f82
20:27:57.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:27:57.184628 4351 raft_consensus.cc:2749] T 00000000000000000000000000000000 P 0db1d539946942f794b0c4ec81f8f4b4 [term 4 FOLLOWER]: Leader pre-election lost for term 5. Reason: could not achieve majority
20:27:57.191 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:57.191 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:57.191 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:57.192 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:57.192 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:57.192 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:57.192 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:57.192 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:57.192 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:57.193 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=221), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [63ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [80ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [119ms] refreshing cache from master, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [121ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [139ms] refreshing cache from master, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [140ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [141ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [219ms] refreshing cache from master, [219ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [221ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@658861865(state=PENDING, result=null, callback=(continuation of Deferred@1631150324 after retry RPC after error@554583590) -> (continuation of Deferred@778746998 after retry RPC after error@584753845) -> (continuation of Deferred@147736794 after retry RPC after error@1528959123) -> (continuation of Deferred@550542989 after retry RPC after error@1466457896) -> (continuation of Deferred@756696455 after retry RPC after error@1727495743) -> (continuation of Deferred@1115140250 after retry RPC after error@1475557091) -> (continuation of Deferred@683309031 after retry RPC after error@1977741897), errback=(continuation of Deferred@1631150324 after retry RPC after error@554583590) -> (continuation of Deferred@778746998 after retry RPC after error@584753845) -> (continuation of Deferred@147736794 after retry RPC after error@1528959123) -> (continuation of Deferred@550542989 after retry RPC after error@1466457896) -> (continuation of Deferred@756696455 after retry RPC after error@1727495743) -> (continuation of Deferred@1115140250 after retry RPC after error@1475557091) -> (continuation of Deferred@683309031 after retry RPC after error@1977741897)))
20:27:57.193 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=221), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [63ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [80ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [119ms] refreshing cache from master, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [121ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [139ms] refreshing cache from master, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [140ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [141ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [219ms] refreshing cache from master, [219ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [221ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@658861865(state=PENDING, result=null, callback=(continuation of Deferred@1631150324 after retry RPC after error@554583590) -> (continuation of Deferred@778746998 after retry RPC after error@584753845) -> (continuation of Deferred@147736794 after retry RPC after error@1528959123) -> (continuation of Deferred@550542989 after retry RPC after error@1466457896) -> (continuation of Deferred@756696455 after retry RPC after error@1727495743) -> (continuation of Deferred@1115140250 after retry RPC after error@1475557091) -> (continuation of Deferred@683309031 after retry RPC after error@1977741897), errback=(continuation of Deferred@1631150324 after retry RPC after error@554583590) -> (continuation of Deferred@778746998 after retry RPC after error@584753845) -> (continuation of Deferred@147736794 after retry RPC after error@1528959123) -> (continuation of Deferred@550542989 after retry RPC after error@1466457896) -> (continuation of Deferred@756696455 after retry RPC after error@1727495743) -> (continuation of Deferred@1115140250 after retry RPC after error@1475557091) -> (continuation of Deferred@683309031 after retry RPC after error@1977741897)))
20:27:57.211 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:57.211 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:57.212 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:57.212 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:57.212 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:57.212 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:57.212 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:57.212 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:57.212 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:57.231 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:57.231 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:57.232 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:57.232 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:57.232 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:57.232 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:57.232 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:57.232 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:57.232 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:57.251 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:57.251 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:57.252 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:57.252 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:57.252 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:57.252 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:57.252 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:57.252 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:57.253 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:57.271 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:57.271 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:57.272 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:57.272 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:57.272 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:57.272 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:57.272 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:57.272 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:57.272 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:57.291 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:57.291 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:57.291 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:57.292 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:57.292 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:57.292 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:57.292 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:57.292 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:57.292 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:57.311 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:57.311 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:57.311 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:57.311 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:57.311 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:57.311 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:57.312 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:57.312 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:57.312 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:57.371 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:57.371 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:57.372 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:57.372 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:57.372 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:57.372 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:57.372 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:57.372 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:57.372 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:57.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:27:57.384850 3893 master_proxy_rpc.cc:177] Failed to determine new leader Master: Timed out: ConnectToClusterRpc(addrs: 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127, num_attempts: 97) passed its deadline: Not found: no leader found: ConnectToClusterRpc(addrs: 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127, num_attempts: 1)
20:27:57.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:27:57.389297 3893 meta_cache.cc:788] Timed out: LookupRpcById { tablet: 'f9ccfdb32e164e61bde68775baab7f9d', attempt: 2 } failed: LookupRpcById timed out after deadline expired: LookupRpcById { tablet: 'f9ccfdb32e164e61bde68775baab7f9d', attempt: 2 } passed its deadline: Timed out: ConnectToClusterRpc(addrs: 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127, num_attempts: 97) passed its deadline: Not found: no leader found: ConnectToClusterRpc(addrs: 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127, num_attempts: 1)
20:27:57.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:27:57.391578 3893 master_proxy_rpc.cc:203] Re-attempting LookupRpcById request to leader Master (127.3.46.62:46071)
20:27:57.411 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:57.411 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:57.411 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:57.412 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:57.412 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:57.412 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:57.412 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:57.412 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:57.412 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:57.413 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [43ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [100ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [161ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [201ms] refreshing cache from master, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, deferred=Deferred@1482373630(state=PENDING, result=null, callback=(continuation of Deferred@755142848 after retry RPC after error@1821773963) -> (continuation of Deferred@1759966098 after retry RPC after error@1777574841) -> (continuation of Deferred@1347332823 after retry RPC after error@209775133) -> (continuation of Deferred@1403384444 after retry RPC after error@658472410) -> (continuation of Deferred@42103800 after retry RPC after error@156873143) -> (continuation of Deferred@1964213490 after retry RPC after error@689890855) -> (continuation of Deferred@949112143 after retry RPC after error@2080540643), errback=(continuation of Deferred@755142848 after retry RPC after error@1821773963) -> (continuation of Deferred@1759966098 after retry RPC after error@1777574841) -> (continuation of Deferred@1347332823 after retry RPC after error@209775133) -> (continuation of Deferred@1403384444 after retry RPC after error@658472410) -> (continuation of Deferred@42103800 after retry RPC after error@156873143) -> (continuation of Deferred@1964213490 after retry RPC after error@689890855) -> (continuation of Deferred@949112143 after retry RPC after error@2080540643)))
20:27:57.413 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [43ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [100ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [161ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [201ms] refreshing cache from master, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, deferred=Deferred@1482373630(state=PENDING, result=null, callback=(continuation of Deferred@755142848 after retry RPC after error@1821773963) -> (continuation of Deferred@1759966098 after retry RPC after error@1777574841) -> (continuation of Deferred@1347332823 after retry RPC after error@209775133) -> (continuation of Deferred@1403384444 after retry RPC after error@658472410) -> (continuation of Deferred@42103800 after retry RPC after error@156873143) -> (continuation of Deferred@1964213490 after retry RPC after error@689890855) -> (continuation of Deferred@949112143 after retry RPC after error@2080540643), errback=(continuation of Deferred@755142848 after retry RPC after error@1821773963) -> (continuation of Deferred@1759966098 after retry RPC after error@1777574841) -> (continuation of Deferred@1347332823 after retry RPC after error@209775133) -> (continuation of Deferred@1403384444 after retry RPC after error@658472410) -> (continuation of Deferred@42103800 after retry RPC after error@156873143) -> (continuation of Deferred@1964213490 after retry RPC after error@689890855) -> (continuation of Deferred@949112143 after retry RPC after error@2080540643)))
20:27:57.431 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:57.431 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:57.431 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:57.432 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:57.432 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:57.432 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:57.432 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:57.432 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:57.432 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:57.451 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:57.451 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:57.451 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:57.451 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:57.451 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:57.452 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:57.452 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:57.452 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:57.452 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:57.471 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:57.471 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:57.472 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:57.472 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:57.472 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:57.472 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:57.472 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:57.472 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:57.472 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:57.491 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:57.491 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:57.492 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:57.492 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:57.492 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:57.492 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:57.492 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:57.492 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:57.492 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:57.511 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:57.511 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:57.511 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:57.511 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:57.512 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:57.512 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:57.512 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:57.512 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:57.512 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:57.551 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:57.551 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:57.552 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:57.552 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:57.552 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:57.552 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:57.552 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:57.552 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:57.552 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:57.591 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:57.592 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:57.592 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:57.592 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:57.592 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:57.592 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:57.592 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:57.592 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:57.593 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:57.610 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:57.610 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:57.611 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:57.611 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:57.611 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:57.611 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:57.611 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:57.611 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:57.612 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:57.612 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=181), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [80ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [161ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [163ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@1633299371(state=PENDING, result=null, callback=(continuation of Deferred@1477682356 after retry RPC after error@1561616945) -> (continuation of Deferred@502343025 after retry RPC after error@767967767) -> (continuation of Deferred@1756718367 after retry RPC after error@374249426) -> (continuation of Deferred@42001095 after retry RPC after error@499764858) -> (continuation of Deferred@1578622273 after retry RPC after error@1015361878) -> (continuation of Deferred@957770459 after retry RPC after error@1786763677) -> (continuation of Deferred@182647878 after retry RPC after error@527987661), errback=(continuation of Deferred@1477682356 after retry RPC after error@1561616945) -> (continuation of Deferred@502343025 after retry RPC after error@767967767) -> (continuation of Deferred@1756718367 after retry RPC after error@374249426) -> (continuation of Deferred@42001095 after retry RPC after error@499764858) -> (continuation of Deferred@1578622273 after retry RPC after error@1015361878) -> (continuation of Deferred@957770459 after retry RPC after error@1786763677) -> (continuation of Deferred@182647878 after retry RPC after error@527987661)))
20:27:57.612 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=181), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [80ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [161ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [163ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@1633299371(state=PENDING, result=null, callback=(continuation of Deferred@1477682356 after retry RPC after error@1561616945) -> (continuation of Deferred@502343025 after retry RPC after error@767967767) -> (continuation of Deferred@1756718367 after retry RPC after error@374249426) -> (continuation of Deferred@42001095 after retry RPC after error@499764858) -> (continuation of Deferred@1578622273 after retry RPC after error@1015361878) -> (continuation of Deferred@957770459 after retry RPC after error@1786763677) -> (continuation of Deferred@182647878 after retry RPC after error@527987661), errback=(continuation of Deferred@1477682356 after retry RPC after error@1561616945) -> (continuation of Deferred@502343025 after retry RPC after error@767967767) -> (continuation of Deferred@1756718367 after retry RPC after error@374249426) -> (continuation of Deferred@42001095 after retry RPC after error@499764858) -> (continuation of Deferred@1578622273 after retry RPC after error@1015361878) -> (continuation of Deferred@957770459 after retry RPC after error@1786763677) -> (continuation of Deferred@182647878 after retry RPC after error@527987661)))
20:27:57.631 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:57.631 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:57.632 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:57.632 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:57.632 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:57.632 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:57.632 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:57.632 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:57.632 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:57.651 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:57.651 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:57.652 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:57.652 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:57.652 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:57.652 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:57.652 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:57.652 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:57.652 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:57.671 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:57.671 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:57.672 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:57.672 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:57.672 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:57.672 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:57.672 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:57.672 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:57.672 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:57.691 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:57.691 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:57.691 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:57.692 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:57.692 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:57.692 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:57.692 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:57.692 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:57.692 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:57.711 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:57.711 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:57.711 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:57.711 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:57.711 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:57.711 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:57.712 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:57.712 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:57.712 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:57.731 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:57.731 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:57.731 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:57.731 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:57.731 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:57.731 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:57.731 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:57.731 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:57.732 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:57.771 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:57.771 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:57.772 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:57.772 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:57.772 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:57.772 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:57.772 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:57.772 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:57.772 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:57.850 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:57.850 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:57.851 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:57.851 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:57.851 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:57.851 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:57.851 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:57.851 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:57.852 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:57.852 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=221), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [1ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [80ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [99ms] refreshing cache from master, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [100ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [100ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [101ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [139ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [141ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [219ms] refreshing cache from master, [219ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [219ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [219ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [220ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [220ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [221ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@1757239104(state=PENDING, result=null, callback=(continuation of Deferred@848868152 after retry RPC after error@1669222933) -> (continuation of Deferred@942130761 after retry RPC after error@975339866) -> (continuation of Deferred@1297409109 after retry RPC after error@1546381584) -> (continuation of Deferred@1522039122 after retry RPC after error@1950227214) -> (continuation of Deferred@505556880 after retry RPC after error@490874413) -> (continuation of Deferred@69844542 after retry RPC after error@4928478) -> (continuation of Deferred@253416168 after retry RPC after error@57015912), errback=(continuation of Deferred@848868152 after retry RPC after error@1669222933) -> (continuation of Deferred@942130761 after retry RPC after error@975339866) -> (continuation of Deferred@1297409109 after retry RPC after error@1546381584) -> (continuation of Deferred@1522039122 after retry RPC after error@1950227214) -> (continuation of Deferred@505556880 after retry RPC after error@490874413) -> (continuation of Deferred@69844542 after retry RPC after error@4928478) -> (continuation of Deferred@253416168 after retry RPC after error@57015912)))
20:27:57.852 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=221), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [1ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [80ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [99ms] refreshing cache from master, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [100ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [100ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [101ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [139ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [141ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [219ms] refreshing cache from master, [219ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [219ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [219ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [220ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [220ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [221ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@1757239104(state=PENDING, result=null, callback=(continuation of Deferred@848868152 after retry RPC after error@1669222933) -> (continuation of Deferred@942130761 after retry RPC after error@975339866) -> (continuation of Deferred@1297409109 after retry RPC after error@1546381584) -> (continuation of Deferred@1522039122 after retry RPC after error@1950227214) -> (continuation of Deferred@505556880 after retry RPC after error@490874413) -> (continuation of Deferred@69844542 after retry RPC after error@4928478) -> (continuation of Deferred@253416168 after retry RPC after error@57015912), errback=(continuation of Deferred@848868152 after retry RPC after error@1669222933) -> (continuation of Deferred@942130761 after retry RPC after error@975339866) -> (continuation of Deferred@1297409109 after retry RPC after error@1546381584) -> (continuation of Deferred@1522039122 after retry RPC after error@1950227214) -> (continuation of Deferred@505556880 after retry RPC after error@490874413) -> (continuation of Deferred@69844542 after retry RPC after error@4928478) -> (continuation of Deferred@253416168 after retry RPC after error@57015912)))
20:27:57.870 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:57.870 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:57.871 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:57.871 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:57.871 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:57.871 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:57.871 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:57.871 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:57.871 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:57.891 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:57.891 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:57.892 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:57.892 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:57.892 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:57.892 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:57.892 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:57.892 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:57.893 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:57.911 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:57.911 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:57.912 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:57.912 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:57.912 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:57.912 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:57.912 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:57.912 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:57.913 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:57.931 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:57.931 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:57.932 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:57.932 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:57.932 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:57.932 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:57.932 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:57.932 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:57.933 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:57.951 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:57.951 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:57.952 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:57.952 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:57.952 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:57.952 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:57.952 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:57.952 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:57.953 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:57.991 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:57.991 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:57.992 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:57.992 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:57.992 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:57.992 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:57.992 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:57.992 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:57.992 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:58.011 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:58.011 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:58.012 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:58.012 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:58.012 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:58.012 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:58.012 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:58.012 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:58.012 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:58.090 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:58.091 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:58.091 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:58.091 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:58.091 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:58.091 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:58.091 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:58.091 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:58.092 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:58.092 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=221), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [63ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [83ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [141ms] refreshing cache from master, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, deferred=Deferred@256424143(state=PENDING, result=null, callback=(continuation of Deferred@1608067038 after retry RPC after error@1988996077) -> (continuation of Deferred@580038110 after retry RPC after error@1758914435) -> (continuation of Deferred@2124925584 after retry RPC after error@1701762154) -> (continuation of Deferred@1378187588 after retry RPC after error@227571324) -> (continuation of Deferred@1294196244 after retry RPC after error@1593675358) -> (continuation of Deferred@669721500 after retry RPC after error@700879499) -> (continuation of Deferred@2059811545 after retry RPC after error@1216074913), errback=(continuation of Deferred@1608067038 after retry RPC after error@1988996077) -> (continuation of Deferred@580038110 after retry RPC after error@1758914435) -> (continuation of Deferred@2124925584 after retry RPC after error@1701762154) -> (continuation of Deferred@1378187588 after retry RPC after error@227571324) -> (continuation of Deferred@1294196244 after retry RPC after error@1593675358) -> (continuation of Deferred@669721500 after retry RPC after error@700879499) -> (continuation of Deferred@2059811545 after retry RPC after error@1216074913)))
20:27:58.092 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=221), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [63ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [83ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [141ms] refreshing cache from master, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, deferred=Deferred@256424143(state=PENDING, result=null, callback=(continuation of Deferred@1608067038 after retry RPC after error@1988996077) -> (continuation of Deferred@580038110 after retry RPC after error@1758914435) -> (continuation of Deferred@2124925584 after retry RPC after error@1701762154) -> (continuation of Deferred@1378187588 after retry RPC after error@227571324) -> (continuation of Deferred@1294196244 after retry RPC after error@1593675358) -> (continuation of Deferred@669721500 after retry RPC after error@700879499) -> (continuation of Deferred@2059811545 after retry RPC after error@1216074913), errback=(continuation of Deferred@1608067038 after retry RPC after error@1988996077) -> (continuation of Deferred@580038110 after retry RPC after error@1758914435) -> (continuation of Deferred@2124925584 after retry RPC after error@1701762154) -> (continuation of Deferred@1378187588 after retry RPC after error@227571324) -> (continuation of Deferred@1294196244 after retry RPC after error@1593675358) -> (continuation of Deferred@669721500 after retry RPC after error@700879499) -> (continuation of Deferred@2059811545 after retry RPC after error@1216074913)))
20:27:58.110 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:58.111 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:58.111 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:58.111 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:58.111 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:58.111 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:58.111 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:58.112 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:58.112 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:58.130 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:58.130 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:58.131 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:58.131 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:58.131 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:58.131 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:58.131 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:58.131 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:58.132 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:58.151 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:58.151 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:58.152 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:58.152 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:58.152 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:58.152 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:58.152 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:58.152 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:58.152 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:58.171 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:58.171 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:58.172 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:58.172 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:58.172 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:58.172 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:58.172 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:58.172 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:58.173 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:58.191 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:58.191 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:58.191 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:58.192 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:58.192 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:58.192 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:58.192 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:58.192 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:58.192 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:58.211 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:58.211 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:58.211 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:58.211 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:58.211 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:58.211 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:58.211 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:58.211 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:58.212 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:58.251 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:58.251 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:58.252 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:58.252 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:58.252 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:58.253 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:58.253 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:58.253 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:58.253 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:58.291 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:58.291 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:58.292 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:58.292 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:58.292 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:58.292 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:58.292 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:58.292 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:58.293 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:58.351 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:58.351 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:58.352 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:58.352 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:58.352 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:58.352 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:58.352 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:58.352 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:58.353 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:58.353 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [143ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [143ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [181ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [241ms] refreshing cache from master, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [243ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [243ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@1471335446(state=PENDING, result=null, callback=(continuation of Deferred@401917833 after retry RPC after error@1137160789) -> (continuation of Deferred@1092219128 after retry RPC after error@677553) -> (continuation of Deferred@829620662 after retry RPC after error@1331185879) -> (continuation of Deferred@1095299060 after retry RPC after error@321230514) -> (continuation of Deferred@988961745 after retry RPC after error@929145512) -> (continuation of Deferred@835771647 after retry RPC after error@243227761) -> (continuation of Deferred@645640892 after retry RPC after error@473247801) -> (continuation of Deferred@724906084 after retry RPC after error@1608347047), errback=(continuation of Deferred@401917833 after retry RPC after error@1137160789) -> (continuation of Deferred@1092219128 after retry RPC after error@677553) -> (continuation of Deferred@829620662 after retry RPC after error@1331185879) -> (continuation of Deferred@1095299060 after retry RPC after error@321230514) -> (continuation of Deferred@988961745 after retry RPC after error@929145512) -> (continuation of Deferred@835771647 after retry RPC after error@243227761) -> (continuation of Deferred@645640892 after retry RPC after error@473247801) -> (continuation of Deferred@724906084 after retry RPC after error@1608347047)))
20:27:58.353 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [143ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [143ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [181ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [241ms] refreshing cache from master, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [243ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [243ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@1471335446(state=PENDING, result=null, callback=(continuation of Deferred@401917833 after retry RPC after error@1137160789) -> (continuation of Deferred@1092219128 after retry RPC after error@677553) -> (continuation of Deferred@829620662 after retry RPC after error@1331185879) -> (continuation of Deferred@1095299060 after retry RPC after error@321230514) -> (continuation of Deferred@988961745 after retry RPC after error@929145512) -> (continuation of Deferred@835771647 after retry RPC after error@243227761) -> (continuation of Deferred@645640892 after retry RPC after error@473247801) -> (continuation of Deferred@724906084 after retry RPC after error@1608347047), errback=(continuation of Deferred@401917833 after retry RPC after error@1137160789) -> (continuation of Deferred@1092219128 after retry RPC after error@677553) -> (continuation of Deferred@829620662 after retry RPC after error@1331185879) -> (continuation of Deferred@1095299060 after retry RPC after error@321230514) -> (continuation of Deferred@988961745 after retry RPC after error@929145512) -> (continuation of Deferred@835771647 after retry RPC after error@243227761) -> (continuation of Deferred@645640892 after retry RPC after error@473247801) -> (continuation of Deferred@724906084 after retry RPC after error@1608347047)))
20:27:58.371 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:58.371 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:58.372 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:58.372 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:58.372 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:58.372 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:58.372 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:58.372 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:58.372 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:58.391 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:58.391 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:58.392 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:58.392 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:58.392 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:58.392 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:58.392 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:58.392 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:58.392 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:58.411 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:58.411 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:58.412 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:58.412 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:58.412 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:58.412 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:58.412 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:58.412 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:58.412 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:58.431 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:58.431 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:58.432 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:58.432 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:58.432 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:58.432 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:58.432 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:58.432 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:58.433 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:58.451 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:58.451 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:58.452 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:58.452 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:58.452 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:58.452 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:58.452 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:58.452 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:58.452 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:58.491 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:58.491 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:58.492 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:58.492 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:58.492 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:58.492 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:58.492 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:58.492 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:58.493 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:58.531 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:58.531 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:58.532 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:58.532 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:58.532 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:58.532 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:58.532 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:58.532 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:58.533 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:58.551 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:58.551 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:58.552 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:58.552 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:58.552 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:58.552 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:58.552 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:58.552 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:58.553 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:58.553 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [1ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@1158054461(state=PENDING, result=null, callback=(continuation of Deferred@411407909 after retry RPC after error@1656181532) -> (continuation of Deferred@196001505 after retry RPC after error@2059981507) -> (continuation of Deferred@1313359617 after retry RPC after error@373281383) -> (continuation of Deferred@1754808311 after retry RPC after error@1417328365) -> (continuation of Deferred@1476821804 after retry RPC after error@809684641) -> (continuation of Deferred@420231385 after retry RPC after error@406862027) -> (continuation of Deferred@1949550348 after retry RPC after error@1386363117), errback=(continuation of Deferred@411407909 after retry RPC after error@1656181532) -> (continuation of Deferred@196001505 after retry RPC after error@2059981507) -> (continuation of Deferred@1313359617 after retry RPC after error@373281383) -> (continuation of Deferred@1754808311 after retry RPC after error@1417328365) -> (continuation of Deferred@1476821804 after retry RPC after error@809684641) -> (continuation of Deferred@420231385 after retry RPC after error@406862027) -> (continuation of Deferred@1949550348 after retry RPC after error@1386363117)))
20:27:58.553 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [1ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@1158054461(state=PENDING, result=null, callback=(continuation of Deferred@411407909 after retry RPC after error@1656181532) -> (continuation of Deferred@196001505 after retry RPC after error@2059981507) -> (continuation of Deferred@1313359617 after retry RPC after error@373281383) -> (continuation of Deferred@1754808311 after retry RPC after error@1417328365) -> (continuation of Deferred@1476821804 after retry RPC after error@809684641) -> (continuation of Deferred@420231385 after retry RPC after error@406862027) -> (continuation of Deferred@1949550348 after retry RPC after error@1386363117), errback=(continuation of Deferred@411407909 after retry RPC after error@1656181532) -> (continuation of Deferred@196001505 after retry RPC after error@2059981507) -> (continuation of Deferred@1313359617 after retry RPC after error@373281383) -> (continuation of Deferred@1754808311 after retry RPC after error@1417328365) -> (continuation of Deferred@1476821804 after retry RPC after error@809684641) -> (continuation of Deferred@420231385 after retry RPC after error@406862027) -> (continuation of Deferred@1949550348 after retry RPC after error@1386363117)))
20:27:58.571 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:58.571 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:58.572 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:58.572 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:58.572 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:58.572 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:58.572 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:58.572 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:58.572 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:58.591 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:58.591 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:58.592 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:58.592 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:58.592 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:58.592 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:58.592 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:58.592 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:58.592 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:58.611 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:58.611 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:58.612 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:58.612 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:58.612 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:58.612 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:58.612 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:58.612 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:58.612 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:58.631 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:58.631 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:58.631 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:58.632 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:58.632 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:58.632 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:58.632 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:58.632 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:58.633 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:58.651 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:58.651 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:58.651 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:58.651 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:58.652 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:58.652 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:58.652 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:58.652 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:58.652 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:58.671 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:58.671 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:58.672 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:58.672 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:58.672 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:58.672 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:58.672 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:58.672 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:58.672 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:58.711 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:58.712 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:58.712 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:58.712 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:58.712 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:58.712 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:58.712 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:58.712 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:58.712 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:58.750 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:58.751 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:58.751 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:58.751 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:58.751 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:58.751 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:58.751 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:58.751 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:58.752 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:58.752 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=181), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [1ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [59ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [99ms] refreshing cache from master, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [101ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [179ms] refreshing cache from master, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [180ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [180ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [181ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@817279074(state=PENDING, result=null, callback=(continuation of Deferred@2083439022 after retry RPC after error@1474753729) -> (continuation of Deferred@235808889 after retry RPC after error@188660858) -> (continuation of Deferred@26685186 after retry RPC after error@1639416589) -> (continuation of Deferred@512045730 after retry RPC after error@1621528676) -> (continuation of Deferred@1483820109 after retry RPC after error@92472229) -> (continuation of Deferred@841232789 after retry RPC after error@1716892165) -> (continuation of Deferred@882153894 after retry RPC after error@689384122), errback=(continuation of Deferred@2083439022 after retry RPC after error@1474753729) -> (continuation of Deferred@235808889 after retry RPC after error@188660858) -> (continuation of Deferred@26685186 after retry RPC after error@1639416589) -> (continuation of Deferred@512045730 after retry RPC after error@1621528676) -> (continuation of Deferred@1483820109 after retry RPC after error@92472229) -> (continuation of Deferred@841232789 after retry RPC after error@1716892165) -> (continuation of Deferred@882153894 after retry RPC after error@689384122)))
20:27:58.753 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=181), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [1ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [59ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [99ms] refreshing cache from master, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [101ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [179ms] refreshing cache from master, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [180ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [180ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [181ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@817279074(state=PENDING, result=null, callback=(continuation of Deferred@2083439022 after retry RPC after error@1474753729) -> (continuation of Deferred@235808889 after retry RPC after error@188660858) -> (continuation of Deferred@26685186 after retry RPC after error@1639416589) -> (continuation of Deferred@512045730 after retry RPC after error@1621528676) -> (continuation of Deferred@1483820109 after retry RPC after error@92472229) -> (continuation of Deferred@841232789 after retry RPC after error@1716892165) -> (continuation of Deferred@882153894 after retry RPC after error@689384122), errback=(continuation of Deferred@2083439022 after retry RPC after error@1474753729) -> (continuation of Deferred@235808889 after retry RPC after error@188660858) -> (continuation of Deferred@26685186 after retry RPC after error@1639416589) -> (continuation of Deferred@512045730 after retry RPC after error@1621528676) -> (continuation of Deferred@1483820109 after retry RPC after error@92472229) -> (continuation of Deferred@841232789 after retry RPC after error@1716892165) -> (continuation of Deferred@882153894 after retry RPC after error@689384122)))
20:27:58.770 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:58.771 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:58.771 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:58.771 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:58.771 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:58.771 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:58.771 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:58.771 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:58.772 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:58.790 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:58.791 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:58.791 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:58.791 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:58.791 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:58.791 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:58.791 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:58.791 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:58.792 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:58.810 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:58.811 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:58.811 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:58.811 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:58.811 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:58.811 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:58.811 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:58.811 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:58.812 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:58.830 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:58.831 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:58.831 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:58.831 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:58.831 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:58.831 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:58.831 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:58.831 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:58.831 [WARN - kudu-nio-3] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:58.850 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:58.850 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:58.851 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:58.851 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:58.851 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:58.851 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:58.851 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:58.851 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:58.852 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:58.870 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:58.870 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:58.871 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:58.871 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:58.871 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:58.871 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:58.871 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:58.871 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:58.872 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:58.910 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:58.911 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:58.911 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:58.911 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:58.911 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:58.911 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:58.911 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:58.911 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:58.912 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:58.950 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:58.951 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:58.951 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:58.951 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:58.951 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:58.951 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:58.951 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:58.951 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:58.952 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:58.952 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=181), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [61ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@733705221(state=PENDING, result=null, callback=(continuation of Deferred@1212696346 after retry RPC after error@1960633974) -> (continuation of Deferred@1295838521 after retry RPC after error@1037546439) -> (continuation of Deferred@925717488 after retry RPC after error@617741365) -> (continuation of Deferred@246323379 after retry RPC after error@241650474) -> (continuation of Deferred@1393044421 after retry RPC after error@2014533061) -> (continuation of Deferred@42082676 after retry RPC after error@914883605) -> (continuation of Deferred@608982786 after retry RPC after error@1184072499), errback=(continuation of Deferred@1212696346 after retry RPC after error@1960633974) -> (continuation of Deferred@1295838521 after retry RPC after error@1037546439) -> (continuation of Deferred@925717488 after retry RPC after error@617741365) -> (continuation of Deferred@246323379 after retry RPC after error@241650474) -> (continuation of Deferred@1393044421 after retry RPC after error@2014533061) -> (continuation of Deferred@42082676 after retry RPC after error@914883605) -> (continuation of Deferred@608982786 after retry RPC after error@1184072499)))
20:27:58.953 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=181), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [61ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@733705221(state=PENDING, result=null, callback=(continuation of Deferred@1212696346 after retry RPC after error@1960633974) -> (continuation of Deferred@1295838521 after retry RPC after error@1037546439) -> (continuation of Deferred@925717488 after retry RPC after error@617741365) -> (continuation of Deferred@246323379 after retry RPC after error@241650474) -> (continuation of Deferred@1393044421 after retry RPC after error@2014533061) -> (continuation of Deferred@42082676 after retry RPC after error@914883605) -> (continuation of Deferred@608982786 after retry RPC after error@1184072499), errback=(continuation of Deferred@1212696346 after retry RPC after error@1960633974) -> (continuation of Deferred@1295838521 after retry RPC after error@1037546439) -> (continuation of Deferred@925717488 after retry RPC after error@617741365) -> (continuation of Deferred@246323379 after retry RPC after error@241650474) -> (continuation of Deferred@1393044421 after retry RPC after error@2014533061) -> (continuation of Deferred@42082676 after retry RPC after error@914883605) -> (continuation of Deferred@608982786 after retry RPC after error@1184072499)))
20:27:58.970 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:58.971 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:58.971 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:58.971 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:58.971 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:58.971 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:58.971 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:58.971 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:58.972 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:58.990 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:58.990 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:58.991 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:58.991 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:58.991 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:58.991 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:58.991 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:58.991 [INFO - pool-40-thread-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:58.992 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:59.011 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:59.011 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:59.012 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:59.012 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:59.012 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:59.012 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:59.012 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:59.012 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:59.013 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:59.030 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:59.031 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:59.031 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:59.031 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:59.031 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:59.031 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:59.031 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:59.031 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:59.032 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:59.050 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:59.050 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:59.051 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:59.051 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:59.051 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:59.051 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:59.051 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:59.051 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:59.052 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:59.071 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:59.071 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:59.072 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:59.072 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:59.072 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:59.072 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:59.072 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:59.072 [INFO - pool-40-thread-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:59.073 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:59.130 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:59.130 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:59.131 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:59.131 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:59.131 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:59.131 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:59.131 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:59.132 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:59.132 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:59.210 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:59.211 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:59.211 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:59.211 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:59.211 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:59.211 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:59.211 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:59.211 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:59.212 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:59.212 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [43ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [103ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@966612032(state=PENDING, result=null, callback=(continuation of Deferred@1969571314 after retry RPC after error@1736399990) -> (continuation of Deferred@698342621 after retry RPC after error@2062791418) -> (continuation of Deferred@627450499 after retry RPC after error@946292405) -> (continuation of Deferred@1875446864 after retry RPC after error@139073329) -> (continuation of Deferred@481058433 after retry RPC after error@519893327) -> (continuation of Deferred@972080460 after retry RPC after error@1633398492) -> (continuation of Deferred@536718368 after retry RPC after error@1727152153), errback=(continuation of Deferred@1969571314 after retry RPC after error@1736399990) -> (continuation of Deferred@698342621 after retry RPC after error@2062791418) -> (continuation of Deferred@627450499 after retry RPC after error@946292405) -> (continuation of Deferred@1875446864 after retry RPC after error@139073329) -> (continuation of Deferred@481058433 after retry RPC after error@519893327) -> (continuation of Deferred@972080460 after retry RPC after error@1633398492) -> (continuation of Deferred@536718368 after retry RPC after error@1727152153)))
20:27:59.213 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [43ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [103ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@966612032(state=PENDING, result=null, callback=(continuation of Deferred@1969571314 after retry RPC after error@1736399990) -> (continuation of Deferred@698342621 after retry RPC after error@2062791418) -> (continuation of Deferred@627450499 after retry RPC after error@946292405) -> (continuation of Deferred@1875446864 after retry RPC after error@139073329) -> (continuation of Deferred@481058433 after retry RPC after error@519893327) -> (continuation of Deferred@972080460 after retry RPC after error@1633398492) -> (continuation of Deferred@536718368 after retry RPC after error@1727152153), errback=(continuation of Deferred@1969571314 after retry RPC after error@1736399990) -> (continuation of Deferred@698342621 after retry RPC after error@2062791418) -> (continuation of Deferred@627450499 after retry RPC after error@946292405) -> (continuation of Deferred@1875446864 after retry RPC after error@139073329) -> (continuation of Deferred@481058433 after retry RPC after error@519893327) -> (continuation of Deferred@972080460 after retry RPC after error@1633398492) -> (continuation of Deferred@536718368 after retry RPC after error@1727152153)))
20:27:59.230 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:59.231 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:59.231 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:59.231 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:59.231 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:59.231 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:59.231 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:59.231 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:59.232 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:59.250 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:59.251 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:59.251 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:59.251 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:59.251 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:59.251 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:59.251 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:59.251 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:59.252 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:59.270 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:59.271 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:59.271 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:59.271 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:59.271 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:59.271 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:59.271 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:59.272 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:59.272 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:59.272 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:59.272 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:59.272 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:59.272 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:59.272 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:59.272 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:59.272 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:59.272 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:59.273 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:59.290 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:59.291 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:59.291 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:59.291 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:59.291 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:59.291 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:59.291 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:59.291 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:59.292 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:59.310 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:59.311 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:59.311 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:59.311 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:59.311 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:59.311 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:59.311 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:59.311 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:59.312 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:59.330 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:59.330 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:59.331 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:59.331 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:59.331 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:59.331 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:59.331 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:59.331 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:59.332 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:59.350 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:59.350 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:59.351 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:59.351 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:59.351 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:59.351 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:59.351 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:59.351 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:59.352 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:59.471 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:59.471 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:59.471 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:59.471 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:59.472 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:59.472 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:59.472 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:59.472 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:59.472 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:59.472 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [42ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [43ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [240ms] refreshing cache from master, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@50384606(state=PENDING, result=null, callback=(continuation of Deferred@1418571531 after retry RPC after error@9181007) -> (continuation of Deferred@319781103 after retry RPC after error@377234030) -> (continuation of Deferred@791687773 after retry RPC after error@1332113051) -> (continuation of Deferred@658943896 after retry RPC after error@1348372044) -> (continuation of Deferred@2119361728 after retry RPC after error@37506855) -> (continuation of Deferred@278141591 after retry RPC after error@886866479) -> (continuation of Deferred@1564542096 after retry RPC after error@1038368104), errback=(continuation of Deferred@1418571531 after retry RPC after error@9181007) -> (continuation of Deferred@319781103 after retry RPC after error@377234030) -> (continuation of Deferred@791687773 after retry RPC after error@1332113051) -> (continuation of Deferred@658943896 after retry RPC after error@1348372044) -> (continuation of Deferred@2119361728 after retry RPC after error@37506855) -> (continuation of Deferred@278141591 after retry RPC after error@886866479) -> (continuation of Deferred@1564542096 after retry RPC after error@1038368104)))
20:27:59.473 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [42ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [43ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [240ms] refreshing cache from master, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@50384606(state=PENDING, result=null, callback=(continuation of Deferred@1418571531 after retry RPC after error@9181007) -> (continuation of Deferred@319781103 after retry RPC after error@377234030) -> (continuation of Deferred@791687773 after retry RPC after error@1332113051) -> (continuation of Deferred@658943896 after retry RPC after error@1348372044) -> (continuation of Deferred@2119361728 after retry RPC after error@37506855) -> (continuation of Deferred@278141591 after retry RPC after error@886866479) -> (continuation of Deferred@1564542096 after retry RPC after error@1038368104), errback=(continuation of Deferred@1418571531 after retry RPC after error@9181007) -> (continuation of Deferred@319781103 after retry RPC after error@377234030) -> (continuation of Deferred@791687773 after retry RPC after error@1332113051) -> (continuation of Deferred@658943896 after retry RPC after error@1348372044) -> (continuation of Deferred@2119361728 after retry RPC after error@37506855) -> (continuation of Deferred@278141591 after retry RPC after error@886866479) -> (continuation of Deferred@1564542096 after retry RPC after error@1038368104)))
20:27:59.491 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:59.491 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:59.491 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:59.491 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:59.491 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:59.491 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:59.492 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:59.492 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:59.492 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:59.511 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:59.511 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:59.511 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:59.511 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:59.511 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:59.511 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:59.512 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:59.512 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:59.512 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:59.531 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:59.531 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:59.531 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:59.531 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:59.532 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:59.532 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:59.532 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:59.532 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:59.532 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:59.551 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:59.551 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:59.551 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:59.552 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:59.552 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:59.552 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:59.552 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:59.552 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:59.552 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:59.571 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:59.571 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:59.571 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:59.571 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:59.571 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:59.572 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:59.572 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:59.572 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:59.572 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:59.611 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:59.611 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:59.611 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:59.611 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:59.611 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:59.612 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:59.612 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:59.612 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:59.612 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:59.690 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:59.691 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:59.691 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:59.691 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:59.691 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:59.691 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:59.691 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:59.691 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:59.692 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:59.692 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=201), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [120ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@1062364821(state=PENDING, result=null, callback=(continuation of Deferred@767410727 after retry RPC after error@1835385647) -> (continuation of Deferred@707900125 after retry RPC after error@117116864) -> (continuation of Deferred@1158961602 after retry RPC after error@111908596) -> (continuation of Deferred@556497830 after retry RPC after error@2138105164) -> (continuation of Deferred@407730473 after retry RPC after error@1116422370) -> (continuation of Deferred@948434270 after retry RPC after error@1076182407), errback=(continuation of Deferred@767410727 after retry RPC after error@1835385647) -> (continuation of Deferred@707900125 after retry RPC after error@117116864) -> (continuation of Deferred@1158961602 after retry RPC after error@111908596) -> (continuation of Deferred@556497830 after retry RPC after error@2138105164) -> (continuation of Deferred@407730473 after retry RPC after error@1116422370) -> (continuation of Deferred@948434270 after retry RPC after error@1076182407)))
20:27:59.692 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=201), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [120ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@1062364821(state=PENDING, result=null, callback=(continuation of Deferred@767410727 after retry RPC after error@1835385647) -> (continuation of Deferred@707900125 after retry RPC after error@117116864) -> (continuation of Deferred@1158961602 after retry RPC after error@111908596) -> (continuation of Deferred@556497830 after retry RPC after error@2138105164) -> (continuation of Deferred@407730473 after retry RPC after error@1116422370) -> (continuation of Deferred@948434270 after retry RPC after error@1076182407), errback=(continuation of Deferred@767410727 after retry RPC after error@1835385647) -> (continuation of Deferred@707900125 after retry RPC after error@117116864) -> (continuation of Deferred@1158961602 after retry RPC after error@111908596) -> (continuation of Deferred@556497830 after retry RPC after error@2138105164) -> (continuation of Deferred@407730473 after retry RPC after error@1116422370) -> (continuation of Deferred@948434270 after retry RPC after error@1076182407)))
20:27:59.710 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:59.710 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:59.711 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:59.711 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:59.711 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:59.711 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:59.711 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:59.711 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:59.712 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:59.731 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:59.731 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:59.732 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:59.732 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:59.732 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:59.732 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:59.732 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:59.732 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:59.732 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:59.750 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:59.751 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:59.751 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:59.751 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:59.751 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:59.759 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:59.759 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:59.759 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:59.759 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:59.770 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:59.771 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:59.771 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:59.771 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:59.771 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:59.771 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:59.771 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:59.771 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:59.772 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:59.790 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:59.791 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:59.791 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:59.791 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:59.791 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:59.791 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:59.791 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:59.791 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:59.792 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:59.810 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:59.811 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:59.811 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:59.811 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:59.811 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:59.811 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:59.811 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:59.811 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:59.812 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:59.870 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:59.871 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:59.871 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:59.871 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:59.871 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:59.871 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:59.871 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:59.871 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:59.872 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:59.872 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=161), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [49ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [49ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@1656600597(state=PENDING, result=null, callback=(continuation of Deferred@2075161171 after retry RPC after error@875089202) -> (continuation of Deferred@339157534 after retry RPC after error@465919389) -> (continuation of Deferred@1671403100 after retry RPC after error@288130624) -> (continuation of Deferred@1009148329 after retry RPC after error@222226949) -> (continuation of Deferred@311206024 after retry RPC after error@1734818335) -> (continuation of Deferred@1757619315 after retry RPC after error@316114348), errback=(continuation of Deferred@2075161171 after retry RPC after error@875089202) -> (continuation of Deferred@339157534 after retry RPC after error@465919389) -> (continuation of Deferred@1671403100 after retry RPC after error@288130624) -> (continuation of Deferred@1009148329 after retry RPC after error@222226949) -> (continuation of Deferred@311206024 after retry RPC after error@1734818335) -> (continuation of Deferred@1757619315 after retry RPC after error@316114348)))
20:27:59.872 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=161), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [49ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [49ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@1656600597(state=PENDING, result=null, callback=(continuation of Deferred@2075161171 after retry RPC after error@875089202) -> (continuation of Deferred@339157534 after retry RPC after error@465919389) -> (continuation of Deferred@1671403100 after retry RPC after error@288130624) -> (continuation of Deferred@1009148329 after retry RPC after error@222226949) -> (continuation of Deferred@311206024 after retry RPC after error@1734818335) -> (continuation of Deferred@1757619315 after retry RPC after error@316114348), errback=(continuation of Deferred@2075161171 after retry RPC after error@875089202) -> (continuation of Deferred@339157534 after retry RPC after error@465919389) -> (continuation of Deferred@1671403100 after retry RPC after error@288130624) -> (continuation of Deferred@1009148329 after retry RPC after error@222226949) -> (continuation of Deferred@311206024 after retry RPC after error@1734818335) -> (continuation of Deferred@1757619315 after retry RPC after error@316114348)))
20:27:59.890 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:59.891 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:59.891 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:59.891 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:59.891 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:59.891 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:59.891 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:59.891 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:59.892 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:59.910 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:59.911 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:59.911 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:59.911 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:59.911 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:59.911 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:59.911 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:59.911 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:59.912 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:59.930 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:59.931 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:59.931 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:59.931 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:59.931 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:59.931 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:59.931 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:59.931 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:59.932 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:59.950 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:59.951 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:59.951 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:59.951 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:59.951 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:59.951 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:59.951 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:59.951 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:59.952 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:59.970 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:59.971 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:59.971 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:59.971 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:59.971 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:59.971 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:59.971 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:59.971 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:59.972 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:27:59.990 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:27:59.991 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:27:59.991 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:59.991 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:59.991 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:27:59.991 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:59.992 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:59.992 [INFO - pool-40-thread-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:27:59.992 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:00.010 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:00.012 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:00.013 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:00.013 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:00.013 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:00.013 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:00.013 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:00.013 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:00.013 [WARN - kudu-nio-7] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:00.030 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:00.031 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:00.031 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:00.031 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:00.031 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:00.031 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:00.031 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:00.031 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:00.032 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:00.070 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:00.071 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:00.071 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:00.071 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:00.071 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:00.071 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:00.071 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:00.072 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:00.072 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:00.072 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=181), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@1907606749(state=PENDING, result=null, callback=(continuation of Deferred@1036334463 after retry RPC after error@2144942046) -> (continuation of Deferred@756792546 after retry RPC after error@1122042032) -> (continuation of Deferred@1444653918 after retry RPC after error@1615466884) -> (continuation of Deferred@273046979 after retry RPC after error@1525169147) -> (continuation of Deferred@1369582243 after retry RPC after error@1641338658) -> (continuation of Deferred@1130243742 after retry RPC after error@1483926253) -> (continuation of Deferred@415577202 after retry RPC after error@1609534863), errback=(continuation of Deferred@1036334463 after retry RPC after error@2144942046) -> (continuation of Deferred@756792546 after retry RPC after error@1122042032) -> (continuation of Deferred@1444653918 after retry RPC after error@1615466884) -> (continuation of Deferred@273046979 after retry RPC after error@1525169147) -> (continuation of Deferred@1369582243 after retry RPC after error@1641338658) -> (continuation of Deferred@1130243742 after retry RPC after error@1483926253) -> (continuation of Deferred@415577202 after retry RPC after error@1609534863)))
20:28:00.073 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=181), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@1907606749(state=PENDING, result=null, callback=(continuation of Deferred@1036334463 after retry RPC after error@2144942046) -> (continuation of Deferred@756792546 after retry RPC after error@1122042032) -> (continuation of Deferred@1444653918 after retry RPC after error@1615466884) -> (continuation of Deferred@273046979 after retry RPC after error@1525169147) -> (continuation of Deferred@1369582243 after retry RPC after error@1641338658) -> (continuation of Deferred@1130243742 after retry RPC after error@1483926253) -> (continuation of Deferred@415577202 after retry RPC after error@1609534863), errback=(continuation of Deferred@1036334463 after retry RPC after error@2144942046) -> (continuation of Deferred@756792546 after retry RPC after error@1122042032) -> (continuation of Deferred@1444653918 after retry RPC after error@1615466884) -> (continuation of Deferred@273046979 after retry RPC after error@1525169147) -> (continuation of Deferred@1369582243 after retry RPC after error@1641338658) -> (continuation of Deferred@1130243742 after retry RPC after error@1483926253) -> (continuation of Deferred@415577202 after retry RPC after error@1609534863)))
20:28:00.090 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:00.091 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:00.091 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:00.091 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:00.091 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:00.091 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:00.091 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:00.092 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:00.092 [WARN - kudu-nio-5] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:00.111 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:00.111 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:00.112 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:00.112 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:00.112 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:00.112 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:00.112 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:00.112 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:00.113 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:00.131 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:00.131 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:00.132 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:00.132 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:00.132 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:00.132 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:00.132 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:00.132 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:00.132 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:00.151 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:00.151 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:00.152 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:00.152 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:00.152 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:00.152 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:00.152 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:00.152 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:00.152 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:00.171 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:00.171 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:00.172 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:00.172 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:00.172 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:00.172 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:00.172 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:00.172 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:00.172 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:00.191 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:00.191 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:00.192 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:00.192 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:00.192 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:00.192 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:00.192 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:00.192 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:00.192 [WARN - kudu-nio-7] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:00.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:28:00.193778 4132 master_runner.cc:186] Error getting master registration for 127.3.46.61:39407: Network error: Client connection negotiation failed: client connection to 127.3.46.61:39407: connect: Connection refused (error 111),
20:28:00.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:28:00.193981 4132 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
20:28:00.210 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:00.210 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:00.211 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:00.211 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:00.211 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:00.211 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:00.211 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:00.211 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:00.212 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:00.311 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:00.311 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:00.311 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:00.311 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:00.311 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:00.311 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:00.311 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:00.311 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:00.312 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:00.313 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [23ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [222ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@1687697584(state=PENDING, result=null, callback=(continuation of Deferred@713881160 after retry RPC after error@1811213377) -> (continuation of Deferred@179098320 after retry RPC after error@802436749) -> (continuation of Deferred@1646771081 after retry RPC after error@788387176) -> (continuation of Deferred@1118545973 after retry RPC after error@1279626931) -> (continuation of Deferred@325160769 after retry RPC after error@1400331984) -> (continuation of Deferred@1275244171 after retry RPC after error@743604654) -> (continuation of Deferred@869669090 after retry RPC after error@1514033558), errback=(continuation of Deferred@713881160 after retry RPC after error@1811213377) -> (continuation of Deferred@179098320 after retry RPC after error@802436749) -> (continuation of Deferred@1646771081 after retry RPC after error@788387176) -> (continuation of Deferred@1118545973 after retry RPC after error@1279626931) -> (continuation of Deferred@325160769 after retry RPC after error@1400331984) -> (continuation of Deferred@1275244171 after retry RPC after error@743604654) -> (continuation of Deferred@869669090 after retry RPC after error@1514033558)))
20:28:00.313 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [23ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [222ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@1687697584(state=PENDING, result=null, callback=(continuation of Deferred@713881160 after retry RPC after error@1811213377) -> (continuation of Deferred@179098320 after retry RPC after error@802436749) -> (continuation of Deferred@1646771081 after retry RPC after error@788387176) -> (continuation of Deferred@1118545973 after retry RPC after error@1279626931) -> (continuation of Deferred@325160769 after retry RPC after error@1400331984) -> (continuation of Deferred@1275244171 after retry RPC after error@743604654) -> (continuation of Deferred@869669090 after retry RPC after error@1514033558), errback=(continuation of Deferred@713881160 after retry RPC after error@1811213377) -> (continuation of Deferred@179098320 after retry RPC after error@802436749) -> (continuation of Deferred@1646771081 after retry RPC after error@788387176) -> (continuation of Deferred@1118545973 after retry RPC after error@1279626931) -> (continuation of Deferred@325160769 after retry RPC after error@1400331984) -> (continuation of Deferred@1275244171 after retry RPC after error@743604654) -> (continuation of Deferred@869669090 after retry RPC after error@1514033558)))
20:28:00.330 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:00.331 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:00.331 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:00.331 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:00.331 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:00.331 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:00.331 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:00.331 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:00.332 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:00.350 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:00.351 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:00.351 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:00.351 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:00.351 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:00.351 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:00.351 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:00.351 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:00.352 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:00.370 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:00.370 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:00.371 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:00.371 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:00.371 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:00.371 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:00.371 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:00.371 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:00.372 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:00.391 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:00.391 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:00.392 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:00.392 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:00.393 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:00.393 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:00.393 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:00.393 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:00.394 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:00.410 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:00.411 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:00.411 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:00.411 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:00.411 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:00.411 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:00.411 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:00.411 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:00.412 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:00.430 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:00.431 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:00.431 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:00.431 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:00.431 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:00.431 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:00.431 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:00.431 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:00.432 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:00.491 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:00.491 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:00.491 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:00.491 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:00.491 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:00.491 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:00.491 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:00.491 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:00.492 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:00.531 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:00.531 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:00.531 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:00.531 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:00.532 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:00.532 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:00.532 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:00.532 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:00.532 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:00.533 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [63ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [63ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [64ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [64ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, deferred=Deferred@490197607(state=PENDING, result=null, callback=(continuation of Deferred@367886252 after retry RPC after error@239246166) -> (continuation of Deferred@1865304510 after retry RPC after error@1277758166) -> (continuation of Deferred@851909856 after retry RPC after error@684042693) -> (continuation of Deferred@901474010 after retry RPC after error@826324225) -> (continuation of Deferred@996839381 after retry RPC after error@1046774524) -> (continuation of Deferred@1351069276 after retry RPC after error@1683743216) -> (continuation of Deferred@186699401 after retry RPC after error@298438259), errback=(continuation of Deferred@367886252 after retry RPC after error@239246166) -> (continuation of Deferred@1865304510 after retry RPC after error@1277758166) -> (continuation of Deferred@851909856 after retry RPC after error@684042693) -> (continuation of Deferred@901474010 after retry RPC after error@826324225) -> (continuation of Deferred@996839381 after retry RPC after error@1046774524) -> (continuation of Deferred@1351069276 after retry RPC after error@1683743216) -> (continuation of Deferred@186699401 after retry RPC after error@298438259)))
20:28:00.533 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [63ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [63ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [64ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [64ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, deferred=Deferred@490197607(state=PENDING, result=null, callback=(continuation of Deferred@367886252 after retry RPC after error@239246166) -> (continuation of Deferred@1865304510 after retry RPC after error@1277758166) -> (continuation of Deferred@851909856 after retry RPC after error@684042693) -> (continuation of Deferred@901474010 after retry RPC after error@826324225) -> (continuation of Deferred@996839381 after retry RPC after error@1046774524) -> (continuation of Deferred@1351069276 after retry RPC after error@1683743216) -> (continuation of Deferred@186699401 after retry RPC after error@298438259), errback=(continuation of Deferred@367886252 after retry RPC after error@239246166) -> (continuation of Deferred@1865304510 after retry RPC after error@1277758166) -> (continuation of Deferred@851909856 after retry RPC after error@684042693) -> (continuation of Deferred@901474010 after retry RPC after error@826324225) -> (continuation of Deferred@996839381 after retry RPC after error@1046774524) -> (continuation of Deferred@1351069276 after retry RPC after error@1683743216) -> (continuation of Deferred@186699401 after retry RPC after error@298438259)))
20:28:00.550 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:00.551 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:00.551 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:00.552 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:00.552 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:00.552 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:00.552 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:00.552 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:00.552 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:00.571 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:00.571 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:00.571 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:00.571 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:00.571 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:00.572 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:00.572 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:00.572 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:00.573 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:00.591 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:00.591 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:00.591 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:00.591 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:00.591 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:00.592 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:00.592 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:00.592 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:00.593 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:00.611 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:00.611 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:00.611 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:00.611 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:00.611 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:00.611 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:00.611 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:00.611 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:00.612 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:00.631 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:00.631 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:00.631 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:00.631 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:00.631 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:00.631 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:00.631 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:00.631 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:00.632 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:00.650 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:00.651 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:00.651 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:00.651 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:00.651 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:00.651 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:00.651 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:00.651 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:00.652 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:00.690 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:00.691 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:00.691 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:00.691 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:00.691 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:00.691 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:00.691 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:00.691 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:00.692 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:00.811 [DEBUG - pool-40-thread-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=260), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [23ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@578955360(state=PENDING, result=null, callback=(continuation of Deferred@1261487519 after retry RPC after error@1403894164) -> (continuation of Deferred@234332998 after retry RPC after error@1040534448) -> (continuation of Deferred@1934912978 after retry RPC after error@1300962101) -> (continuation of Deferred@2111208529 after retry RPC after error@1771757712) -> (continuation of Deferred@963652236 after retry RPC after error@750246948) -> (continuation of Deferred@1649219863 after retry RPC after error@1550711726) -> (continuation of Deferred@1685167845 after retry RPC after error@1991588019), errback=(continuation of Deferred@1261487519 after retry RPC after error@1403894164) -> (continuation of Deferred@234332998 after retry RPC after error@1040534448) -> (continuation of Deferred@1934912978 after retry RPC after error@1300962101) -> (continuation of Deferred@2111208529 after retry RPC after error@1771757712) -> (continuation of Deferred@963652236 after retry RPC after error@750246948) -> (continuation of Deferred@1649219863 after retry RPC after error@1550711726) -> (continuation of Deferred@1685167845 after retry RPC after error@1991588019)))
20:28:00.812 [WARN - pool-40-thread-1] (HashedWheelTimer.java:716) An exception was thrown by TimerTask.
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713)
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701)
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:787)
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501)
at java.base/java.lang.Thread.run(Thread.java:840)
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713)
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701)
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:787)
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501)
at java.base/java.lang.Thread.run(Thread.java:840)
20:28:00.830 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:00.831 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:00.831 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:00.831 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:00.831 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:00.831 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:00.831 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:00.831 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:00.832 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:00.850 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:00.851 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:00.851 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:00.851 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:00.851 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:00.851 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:00.851 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:00.851 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:00.852 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:00.870 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:00.871 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:00.871 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:00.871 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:00.871 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:00.871 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:00.871 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:00.871 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:00.872 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:00.890 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:00.891 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:00.891 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:00.891 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:00.891 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:00.891 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:00.891 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:00.891 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:00.892 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:00.910 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:00.911 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:00.911 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:00.911 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:00.911 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:00.911 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:00.911 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:00.911 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:00.912 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:00.930 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:00.931 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:00.931 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:00.931 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:00.931 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:00.931 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:00.931 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:00.931 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:00.932 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:00.950 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:00.951 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:00.951 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:00.951 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:00.951 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:00.951 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:00.951 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:00.951 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:00.952 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:00.990 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:00.991 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:00.991 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:00.991 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:00.991 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:00.991 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:00.991 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:00.991 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:00.992 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:01.071 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:01.071 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:01.071 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:01.071 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:01.071 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:01.072 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:01.072 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:01.072 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:01.073 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:01.073 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [243ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [243ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@644506879(state=PENDING, result=null, callback=(continuation of Deferred@199542639 after retry RPC after error@1546448717) -> (continuation of Deferred@2044223944 after retry RPC after error@207662992) -> (continuation of Deferred@554880716 after retry RPC after error@444533540) -> (continuation of Deferred@546566018 after retry RPC after error@696698927) -> (continuation of Deferred@517281661 after retry RPC after error@2089879976) -> (continuation of Deferred@1128334919 after retry RPC after error@1545169328) -> (continuation of Deferred@1782707121 after retry RPC after error@518613784) -> (continuation of Deferred@1431381436 after retry RPC after error@1116897871), errback=(continuation of Deferred@199542639 after retry RPC after error@1546448717) -> (continuation of Deferred@2044223944 after retry RPC after error@207662992) -> (continuation of Deferred@554880716 after retry RPC after error@444533540) -> (continuation of Deferred@546566018 after retry RPC after error@696698927) -> (continuation of Deferred@517281661 after retry RPC after error@2089879976) -> (continuation of Deferred@1128334919 after retry RPC after error@1545169328) -> (continuation of Deferred@1782707121 after retry RPC after error@518613784) -> (continuation of Deferred@1431381436 after retry RPC after error@1116897871)))
20:28:01.073 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [243ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [243ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@644506879(state=PENDING, result=null, callback=(continuation of Deferred@199542639 after retry RPC after error@1546448717) -> (continuation of Deferred@2044223944 after retry RPC after error@207662992) -> (continuation of Deferred@554880716 after retry RPC after error@444533540) -> (continuation of Deferred@546566018 after retry RPC after error@696698927) -> (continuation of Deferred@517281661 after retry RPC after error@2089879976) -> (continuation of Deferred@1128334919 after retry RPC after error@1545169328) -> (continuation of Deferred@1782707121 after retry RPC after error@518613784) -> (continuation of Deferred@1431381436 after retry RPC after error@1116897871), errback=(continuation of Deferred@199542639 after retry RPC after error@1546448717) -> (continuation of Deferred@2044223944 after retry RPC after error@207662992) -> (continuation of Deferred@554880716 after retry RPC after error@444533540) -> (continuation of Deferred@546566018 after retry RPC after error@696698927) -> (continuation of Deferred@517281661 after retry RPC after error@2089879976) -> (continuation of Deferred@1128334919 after retry RPC after error@1545169328) -> (continuation of Deferred@1782707121 after retry RPC after error@518613784) -> (continuation of Deferred@1431381436 after retry RPC after error@1116897871)))
20:28:01.091 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:01.091 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:01.091 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:01.091 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:01.091 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:01.091 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:01.092 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:01.092 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:01.093 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:01.111 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:01.111 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:01.111 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:01.111 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:01.111 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:01.111 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:01.111 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:01.112 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:01.112 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:01.131 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:01.131 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:01.131 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:01.131 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:01.131 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:01.131 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:01.131 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:01.131 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:01.132 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:01.150 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:01.151 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:01.151 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:01.151 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:01.151 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:01.151 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:01.151 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:01.151 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:01.152 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:01.171 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:01.171 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:01.171 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:01.171 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:01.172 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:01.172 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:01.172 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:01.172 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:01.172 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:01.211 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:01.211 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:01.211 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:01.211 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:01.211 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:01.212 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:01.212 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:01.212 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:01.212 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:01.271 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:01.271 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:01.272 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:01.272 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:01.272 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:01.272 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:01.272 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:01.272 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:01.272 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:01.273 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [3ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [120ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [181ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@232410358(state=PENDING, result=null, callback=(continuation of Deferred@1598167904 after retry RPC after error@1746064626) -> (continuation of Deferred@2141404088 after retry RPC after error@541160774) -> (continuation of Deferred@1863840565 after retry RPC after error@299638625) -> (continuation of Deferred@2009368601 after retry RPC after error@1543780578) -> (continuation of Deferred@215676428 after retry RPC after error@2106382305) -> (continuation of Deferred@786107401 after retry RPC after error@1407574552), errback=(continuation of Deferred@1598167904 after retry RPC after error@1746064626) -> (continuation of Deferred@2141404088 after retry RPC after error@541160774) -> (continuation of Deferred@1863840565 after retry RPC after error@299638625) -> (continuation of Deferred@2009368601 after retry RPC after error@1543780578) -> (continuation of Deferred@215676428 after retry RPC after error@2106382305) -> (continuation of Deferred@786107401 after retry RPC after error@1407574552)))
20:28:01.273 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [3ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [120ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [181ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@232410358(state=PENDING, result=null, callback=(continuation of Deferred@1598167904 after retry RPC after error@1746064626) -> (continuation of Deferred@2141404088 after retry RPC after error@541160774) -> (continuation of Deferred@1863840565 after retry RPC after error@299638625) -> (continuation of Deferred@2009368601 after retry RPC after error@1543780578) -> (continuation of Deferred@215676428 after retry RPC after error@2106382305) -> (continuation of Deferred@786107401 after retry RPC after error@1407574552), errback=(continuation of Deferred@1598167904 after retry RPC after error@1746064626) -> (continuation of Deferred@2141404088 after retry RPC after error@541160774) -> (continuation of Deferred@1863840565 after retry RPC after error@299638625) -> (continuation of Deferred@2009368601 after retry RPC after error@1543780578) -> (continuation of Deferred@215676428 after retry RPC after error@2106382305) -> (continuation of Deferred@786107401 after retry RPC after error@1407574552)))
20:28:01.291 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:01.291 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:01.292 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:01.292 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:01.292 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:01.292 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:01.292 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:01.292 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:01.292 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:01.311 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:01.311 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:01.312 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:01.312 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:01.312 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:01.312 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:01.312 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:01.312 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:01.312 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:01.331 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:01.331 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:01.331 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:01.332 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:01.332 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:01.332 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:01.332 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:01.332 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:01.332 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:01.351 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:01.351 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:01.352 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:01.352 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:01.352 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:01.352 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:01.352 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:01.352 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:01.353 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:01.371 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:01.371 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:01.371 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:01.372 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:01.372 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:01.372 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:01.372 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:01.372 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:01.373 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:01.391 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:01.391 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:01.391 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:01.391 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:01.391 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:01.392 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:01.392 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:01.392 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:01.393 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:01.431 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:01.431 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:01.432 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:01.432 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:01.432 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:01.432 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:01.432 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:01.432 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:01.432 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:01.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:28:01.442463 4387 raft_consensus.cc:493] T 00000000000000000000000000000000 P 0db1d539946942f794b0c4ec81f8f4b4 [term 4 FOLLOWER]: Starting pre-election (detected failure of leader b1731f8308c145f686f88efd6a54b851)
20:28:01.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:28:01.442687 4387 raft_consensus.cc:515] T 00000000000000000000000000000000 P 0db1d539946942f794b0c4ec81f8f4b4 [term 4 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0db1d539946942f794b0c4ec81f8f4b4" member_type: VOTER last_known_addr { host: "127.3.46.62" port: 46071 } } peers { permanent_uuid: "d6c270b387b04f17b2a58bfcb0eb4f82" member_type: VOTER last_known_addr { host: "127.3.46.61" port: 39407 } } peers { permanent_uuid: "b1731f8308c145f686f88efd6a54b851" member_type: VOTER last_known_addr { host: "127.3.46.60" port: 36127 } }
20:28:01.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:28:01.443604 4387 leader_election.cc:290] T 00000000000000000000000000000000 P 0db1d539946942f794b0c4ec81f8f4b4 [CANDIDATE]: Term 5 pre-election: Requested pre-vote from peers d6c270b387b04f17b2a58bfcb0eb4f82 (127.3.46.61:39407), b1731f8308c145f686f88efd6a54b851 (127.3.46.60:36127)
20:28:01.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:28:01.446084 4149 leader_election.cc:336] T 00000000000000000000000000000000 P 0db1d539946942f794b0c4ec81f8f4b4 [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer d6c270b387b04f17b2a58bfcb0eb4f82 (127.3.46.61:39407): Network error: Client connection negotiation failed: client connection to 127.3.46.61:39407: connect: Connection refused (error 111)
20:28:01.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:28:01.446256 4148 leader_election.cc:336] T 00000000000000000000000000000000 P 0db1d539946942f794b0c4ec81f8f4b4 [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer b1731f8308c145f686f88efd6a54b851 (127.3.46.60:36127): Network error: Client connection negotiation failed: client connection to 127.3.46.60:36127: connect: Connection refused (error 111)
20:28:01.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:28:01.446375 4148 leader_election.cc:304] T 00000000000000000000000000000000 P 0db1d539946942f794b0c4ec81f8f4b4 [CANDIDATE]: Term 5 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 0db1d539946942f794b0c4ec81f8f4b4; no voters: b1731f8308c145f686f88efd6a54b851, d6c270b387b04f17b2a58bfcb0eb4f82
20:28:01.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:28:01.446655 4387 raft_consensus.cc:2749] T 00000000000000000000000000000000 P 0db1d539946942f794b0c4ec81f8f4b4 [term 4 FOLLOWER]: Leader pre-election lost for term 5. Reason: could not achieve majority
20:28:01.471 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:01.471 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:01.472 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:01.472 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:01.472 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:01.472 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:01.472 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:01.472 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:01.472 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:01.473 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=181), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [1ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [99ms] refreshing cache from master, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [100ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@1168463494(state=PENDING, result=null, callback=(continuation of Deferred@685677582 after retry RPC after error@2000823710) -> (continuation of Deferred@1490293180 after retry RPC after error@925527366) -> (continuation of Deferred@1799924033 after retry RPC after error@222302051) -> (continuation of Deferred@1977726598 after retry RPC after error@289776929) -> (continuation of Deferred@1072239849 after retry RPC after error@845102944) -> (continuation of Deferred@1249049131 after retry RPC after error@1990337257) -> (continuation of Deferred@26951304 after retry RPC after error@1249158628), errback=(continuation of Deferred@685677582 after retry RPC after error@2000823710) -> (continuation of Deferred@1490293180 after retry RPC after error@925527366) -> (continuation of Deferred@1799924033 after retry RPC after error@222302051) -> (continuation of Deferred@1977726598 after retry RPC after error@289776929) -> (continuation of Deferred@1072239849 after retry RPC after error@845102944) -> (continuation of Deferred@1249049131 after retry RPC after error@1990337257) -> (continuation of Deferred@26951304 after retry RPC after error@1249158628)))
20:28:01.473 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=181), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [1ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [99ms] refreshing cache from master, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [100ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@1168463494(state=PENDING, result=null, callback=(continuation of Deferred@685677582 after retry RPC after error@2000823710) -> (continuation of Deferred@1490293180 after retry RPC after error@925527366) -> (continuation of Deferred@1799924033 after retry RPC after error@222302051) -> (continuation of Deferred@1977726598 after retry RPC after error@289776929) -> (continuation of Deferred@1072239849 after retry RPC after error@845102944) -> (continuation of Deferred@1249049131 after retry RPC after error@1990337257) -> (continuation of Deferred@26951304 after retry RPC after error@1249158628), errback=(continuation of Deferred@685677582 after retry RPC after error@2000823710) -> (continuation of Deferred@1490293180 after retry RPC after error@925527366) -> (continuation of Deferred@1799924033 after retry RPC after error@222302051) -> (continuation of Deferred@1977726598 after retry RPC after error@289776929) -> (continuation of Deferred@1072239849 after retry RPC after error@845102944) -> (continuation of Deferred@1249049131 after retry RPC after error@1990337257) -> (continuation of Deferred@26951304 after retry RPC after error@1249158628)))
20:28:01.491 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:01.491 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:01.491 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:01.491 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:01.491 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:01.492 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:01.492 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:01.492 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:01.492 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:01.511 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:01.511 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:01.511 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:01.511 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:01.512 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:01.512 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:01.512 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:01.512 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:01.512 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:01.531 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:01.531 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:01.531 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:01.531 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:01.531 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:01.531 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:01.532 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:01.532 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:01.532 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:01.551 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:01.551 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:01.551 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:01.551 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:01.551 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:01.551 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:01.552 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:01.552 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:01.552 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:01.571 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:01.571 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:01.571 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:01.571 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:01.571 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:01.571 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:01.571 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:01.571 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:01.572 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:01.591 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:01.591 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:01.591 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:01.592 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:01.592 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:01.592 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:01.592 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:01.592 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:01.592 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:01.611 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:01.611 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:01.611 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:01.612 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:01.612 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:01.612 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:01.612 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:01.612 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:01.612 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:01.730 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:01.731 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:01.731 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:01.731 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:01.731 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:01.731 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:01.731 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:01.731 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:01.732 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:01.732 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=241), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [1ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [19ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [20ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [40ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [60ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [80ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [99ms] refreshing cache from master, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [101ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [119ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [121ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [239ms] refreshing cache from master, [239ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [239ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [240ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [240ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [241ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@100334253(state=PENDING, result=null, callback=(continuation of Deferred@915658939 after retry RPC after error@1708282826) -> (continuation of Deferred@2134181508 after retry RPC after error@768123997) -> (continuation of Deferred@95876959 after retry RPC after error@1913501728) -> (continuation of Deferred@626753987 after retry RPC after error@1306080121) -> (continuation of Deferred@2137467282 after retry RPC after error@1614041830) -> (continuation of Deferred@421036730 after retry RPC after error@492709915) -> (continuation of Deferred@1530112411 after retry RPC after error@433222521), errback=(continuation of Deferred@915658939 after retry RPC after error@1708282826) -> (continuation of Deferred@2134181508 after retry RPC after error@768123997) -> (continuation of Deferred@95876959 after retry RPC after error@1913501728) -> (continuation of Deferred@626753987 after retry RPC after error@1306080121) -> (continuation of Deferred@2137467282 after retry RPC after error@1614041830) -> (continuation of Deferred@421036730 after retry RPC after error@492709915) -> (continuation of Deferred@1530112411 after retry RPC after error@433222521)))
20:28:01.733 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=241), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [1ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [19ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [20ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [40ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [60ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [80ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [99ms] refreshing cache from master, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [101ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [119ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [121ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [239ms] refreshing cache from master, [239ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [239ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [240ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [240ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [241ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@100334253(state=PENDING, result=null, callback=(continuation of Deferred@915658939 after retry RPC after error@1708282826) -> (continuation of Deferred@2134181508 after retry RPC after error@768123997) -> (continuation of Deferred@95876959 after retry RPC after error@1913501728) -> (continuation of Deferred@626753987 after retry RPC after error@1306080121) -> (continuation of Deferred@2137467282 after retry RPC after error@1614041830) -> (continuation of Deferred@421036730 after retry RPC after error@492709915) -> (continuation of Deferred@1530112411 after retry RPC after error@433222521), errback=(continuation of Deferred@915658939 after retry RPC after error@1708282826) -> (continuation of Deferred@2134181508 after retry RPC after error@768123997) -> (continuation of Deferred@95876959 after retry RPC after error@1913501728) -> (continuation of Deferred@626753987 after retry RPC after error@1306080121) -> (continuation of Deferred@2137467282 after retry RPC after error@1614041830) -> (continuation of Deferred@421036730 after retry RPC after error@492709915) -> (continuation of Deferred@1530112411 after retry RPC after error@433222521)))
20:28:01.750 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:01.751 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:01.751 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:01.751 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:01.751 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:01.751 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:01.751 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:01.751 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:01.752 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:01.770 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:01.771 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:01.771 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:01.771 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:01.771 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:01.771 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:01.771 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:01.771 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:01.772 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:01.790 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:01.791 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:01.791 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:01.791 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:01.791 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:01.791 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:01.791 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:01.791 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:01.792 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:01.810 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:01.810 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:01.811 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:01.811 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:01.811 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:01.811 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:01.811 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:01.811 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:01.812 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:01.830 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:01.830 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:01.831 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:01.831 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:01.831 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:01.831 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:01.831 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:01.831 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:01.832 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:01.871 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:01.871 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:01.872 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:01.872 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:01.872 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:01.872 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:01.872 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:01.872 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:01.873 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:01.950 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:01.951 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:01.951 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:01.951 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:01.951 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:01.951 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:01.951 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:01.952 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:01.952 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:01.952 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=201), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [123ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@1858720292(state=PENDING, result=null, callback=(continuation of Deferred@1686781247 after retry RPC after error@824195426) -> (continuation of Deferred@531679455 after retry RPC after error@1061708287) -> (continuation of Deferred@178180332 after retry RPC after error@977605620) -> (continuation of Deferred@590008915 after retry RPC after error@1603344464) -> (continuation of Deferred@594852606 after retry RPC after error@834455511) -> (continuation of Deferred@1156654073 after retry RPC after error@326492335), errback=(continuation of Deferred@1686781247 after retry RPC after error@824195426) -> (continuation of Deferred@531679455 after retry RPC after error@1061708287) -> (continuation of Deferred@178180332 after retry RPC after error@977605620) -> (continuation of Deferred@590008915 after retry RPC after error@1603344464) -> (continuation of Deferred@594852606 after retry RPC after error@834455511) -> (continuation of Deferred@1156654073 after retry RPC after error@326492335)))
20:28:01.953 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=201), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [123ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@1858720292(state=PENDING, result=null, callback=(continuation of Deferred@1686781247 after retry RPC after error@824195426) -> (continuation of Deferred@531679455 after retry RPC after error@1061708287) -> (continuation of Deferred@178180332 after retry RPC after error@977605620) -> (continuation of Deferred@590008915 after retry RPC after error@1603344464) -> (continuation of Deferred@594852606 after retry RPC after error@834455511) -> (continuation of Deferred@1156654073 after retry RPC after error@326492335), errback=(continuation of Deferred@1686781247 after retry RPC after error@824195426) -> (continuation of Deferred@531679455 after retry RPC after error@1061708287) -> (continuation of Deferred@178180332 after retry RPC after error@977605620) -> (continuation of Deferred@590008915 after retry RPC after error@1603344464) -> (continuation of Deferred@594852606 after retry RPC after error@834455511) -> (continuation of Deferred@1156654073 after retry RPC after error@326492335)))
20:28:01.971 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:01.971 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:01.971 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:01.971 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:01.971 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:01.971 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:01.972 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:01.972 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:01.972 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:01.991 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:01.991 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:01.991 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:01.991 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:01.991 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:01.992 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:01.992 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:01.992 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:01.992 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:02.011 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:02.011 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:02.011 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:02.011 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:02.011 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:02.011 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:02.011 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:02.011 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:02.012 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:02.030 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:02.031 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:02.031 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:02.031 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:02.031 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:02.031 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:02.031 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:02.031 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:02.032 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:02.050 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:02.051 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:02.051 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:02.051 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:02.051 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:02.051 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:02.051 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:02.051 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:02.053 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:02.090 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:02.091 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:02.091 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:02.091 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:02.091 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:02.091 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:02.091 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:02.091 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:02.092 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:02.131 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:02.131 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:02.131 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:02.131 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:02.131 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:02.131 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:02.131 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:02.132 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:02.132 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:02.171 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:02.171 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:02.171 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:02.171 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:02.171 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:02.171 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:02.171 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:02.172 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:02.172 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:02.172 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=201), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@645185101(state=PENDING, result=null, callback=(continuation of Deferred@1262473256 after retry RPC after error@1936254779) -> (continuation of Deferred@275162240 after retry RPC after error@325314071) -> (continuation of Deferred@652790467 after retry RPC after error@302882842) -> (continuation of Deferred@938103935 after retry RPC after error@1001742283) -> (continuation of Deferred@1533752492 after retry RPC after error@614023851) -> (continuation of Deferred@1102427115 after retry RPC after error@1349510929) -> (continuation of Deferred@1386444772 after retry RPC after error@1010952961), errback=(continuation of Deferred@1262473256 after retry RPC after error@1936254779) -> (continuation of Deferred@275162240 after retry RPC after error@325314071) -> (continuation of Deferred@652790467 after retry RPC after error@302882842) -> (continuation of Deferred@938103935 after retry RPC after error@1001742283) -> (continuation of Deferred@1533752492 after retry RPC after error@614023851) -> (continuation of Deferred@1102427115 after retry RPC after error@1349510929) -> (continuation of Deferred@1386444772 after retry RPC after error@1010952961)))
20:28:02.173 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=201), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@645185101(state=PENDING, result=null, callback=(continuation of Deferred@1262473256 after retry RPC after error@1936254779) -> (continuation of Deferred@275162240 after retry RPC after error@325314071) -> (continuation of Deferred@652790467 after retry RPC after error@302882842) -> (continuation of Deferred@938103935 after retry RPC after error@1001742283) -> (continuation of Deferred@1533752492 after retry RPC after error@614023851) -> (continuation of Deferred@1102427115 after retry RPC after error@1349510929) -> (continuation of Deferred@1386444772 after retry RPC after error@1010952961), errback=(continuation of Deferred@1262473256 after retry RPC after error@1936254779) -> (continuation of Deferred@275162240 after retry RPC after error@325314071) -> (continuation of Deferred@652790467 after retry RPC after error@302882842) -> (continuation of Deferred@938103935 after retry RPC after error@1001742283) -> (continuation of Deferred@1533752492 after retry RPC after error@614023851) -> (continuation of Deferred@1102427115 after retry RPC after error@1349510929) -> (continuation of Deferred@1386444772 after retry RPC after error@1010952961)))
20:28:02.191 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:02.191 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:02.191 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:02.192 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:02.192 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:02.192 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:02.192 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:02.193 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:02.193 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:02.211 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:02.211 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:02.211 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:02.211 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:02.211 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:02.212 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:02.212 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:02.212 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:02.212 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:02.231 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:02.231 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:02.231 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:02.231 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:02.231 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:02.231 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:02.231 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:02.232 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:02.232 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:02.251 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:02.252 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:02.252 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:02.252 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:02.252 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:02.252 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:02.252 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:02.252 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:02.253 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:02.270 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:02.271 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:02.271 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:02.271 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:02.271 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:02.271 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:02.271 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:02.271 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:02.272 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:02.290 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:02.291 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:02.291 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:02.291 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:02.291 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:02.291 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:02.291 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:02.291 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:02.292 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:02.330 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:02.331 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:02.331 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:02.331 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:02.331 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:02.331 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:02.331 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:02.331 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:02.332 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:02.411 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:02.411 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:02.411 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:02.411 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:02.411 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:02.411 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:02.412 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:02.412 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:02.412 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:02.412 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [3ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [62ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [63ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [220ms] refreshing cache from master, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [222ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [222ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [222ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@1812113856(state=PENDING, result=null, callback=(continuation of Deferred@1795523344 after retry RPC after error@261515992) -> (continuation of Deferred@262832225 after retry RPC after error@1990161661) -> (continuation of Deferred@826635325 after retry RPC after error@230234215) -> (continuation of Deferred@1456158777 after retry RPC after error@1543407233) -> (continuation of Deferred@1090577759 after retry RPC after error@79669048) -> (continuation of Deferred@188374374 after retry RPC after error@618596366) -> (continuation of Deferred@1647520727 after retry RPC after error@458016462), errback=(continuation of Deferred@1795523344 after retry RPC after error@261515992) -> (continuation of Deferred@262832225 after retry RPC after error@1990161661) -> (continuation of Deferred@826635325 after retry RPC after error@230234215) -> (continuation of Deferred@1456158777 after retry RPC after error@1543407233) -> (continuation of Deferred@1090577759 after retry RPC after error@79669048) -> (continuation of Deferred@188374374 after retry RPC after error@618596366) -> (continuation of Deferred@1647520727 after retry RPC after error@458016462)))
20:28:02.413 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [3ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [62ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [63ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [220ms] refreshing cache from master, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [222ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [222ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [222ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@1812113856(state=PENDING, result=null, callback=(continuation of Deferred@1795523344 after retry RPC after error@261515992) -> (continuation of Deferred@262832225 after retry RPC after error@1990161661) -> (continuation of Deferred@826635325 after retry RPC after error@230234215) -> (continuation of Deferred@1456158777 after retry RPC after error@1543407233) -> (continuation of Deferred@1090577759 after retry RPC after error@79669048) -> (continuation of Deferred@188374374 after retry RPC after error@618596366) -> (continuation of Deferred@1647520727 after retry RPC after error@458016462), errback=(continuation of Deferred@1795523344 after retry RPC after error@261515992) -> (continuation of Deferred@262832225 after retry RPC after error@1990161661) -> (continuation of Deferred@826635325 after retry RPC after error@230234215) -> (continuation of Deferred@1456158777 after retry RPC after error@1543407233) -> (continuation of Deferred@1090577759 after retry RPC after error@79669048) -> (continuation of Deferred@188374374 after retry RPC after error@618596366) -> (continuation of Deferred@1647520727 after retry RPC after error@458016462)))
20:28:02.431 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:02.431 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:02.431 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:02.431 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:02.431 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:02.432 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:02.432 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:02.432 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:02.432 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:02.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:28:02.447981 3893 master_proxy_rpc.cc:177] Failed to determine new leader Master: Timed out: ConnectToClusterRpc(addrs: 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127, num_attempts: 97) passed its deadline: Not found: no leader found: ConnectToClusterRpc(addrs: 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127, num_attempts: 1)
20:28:02.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:28:02.449458 3893 meta_cache.cc:788] Timed out: LookupRpcById { tablet: 'f9ccfdb32e164e61bde68775baab7f9d', attempt: 2 } failed: LookupRpcById timed out after deadline expired: LookupRpcById { tablet: 'f9ccfdb32e164e61bde68775baab7f9d', attempt: 2 } passed its deadline: Timed out: ConnectToClusterRpc(addrs: 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127, num_attempts: 97) passed its deadline: Not found: no leader found: ConnectToClusterRpc(addrs: 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127, num_attempts: 1)
20:28:02.451 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:02.451 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:02.451 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:02.452 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:02.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:28:02.451882 3893 master_proxy_rpc.cc:203] Re-attempting LookupRpcById request to leader Master (127.3.46.62:46071)
20:28:02.451 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:02.452 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:02.452 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:02.452 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:02.453 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:02.471 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:02.471 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:02.471 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:02.471 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:02.471 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:02.471 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:02.471 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:02.471 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:02.472 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:02.491 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:02.491 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:02.491 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:02.491 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:02.491 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:02.491 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:02.492 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:02.492 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:02.492 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:02.511 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:02.511 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:02.511 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:02.511 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:02.512 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:02.512 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:02.512 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:02.512 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:02.512 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:02.551 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:02.551 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:02.551 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:02.551 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:02.551 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:02.551 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:02.551 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:02.551 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:02.552 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:02.611 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:02.611 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:02.612 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:02.612 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:02.612 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:02.612 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:02.612 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:02.612 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:02.613 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:02.671 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:02.671 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:02.671 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:02.671 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:02.671 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:02.672 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:02.672 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:02.672 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:02.672 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:02.673 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=241), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [23ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [181ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [183ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [184ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, deferred=Deferred@1534005824(state=PENDING, result=null, callback=(continuation of Deferred@902302560 after retry RPC after error@405610190) -> (continuation of Deferred@278417616 after retry RPC after error@868658577) -> (continuation of Deferred@1176515458 after retry RPC after error@915115750) -> (continuation of Deferred@1690781239 after retry RPC after error@1651229979) -> (continuation of Deferred@630216439 after retry RPC after error@1325797405) -> (continuation of Deferred@296620340 after retry RPC after error@768691463) -> (continuation of Deferred@173113764 after retry RPC after error@1974005259), errback=(continuation of Deferred@902302560 after retry RPC after error@405610190) -> (continuation of Deferred@278417616 after retry RPC after error@868658577) -> (continuation of Deferred@1176515458 after retry RPC after error@915115750) -> (continuation of Deferred@1690781239 after retry RPC after error@1651229979) -> (continuation of Deferred@630216439 after retry RPC after error@1325797405) -> (continuation of Deferred@296620340 after retry RPC after error@768691463) -> (continuation of Deferred@173113764 after retry RPC after error@1974005259)))
20:28:02.673 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=241), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [23ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [181ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [183ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [184ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, deferred=Deferred@1534005824(state=PENDING, result=null, callback=(continuation of Deferred@902302560 after retry RPC after error@405610190) -> (continuation of Deferred@278417616 after retry RPC after error@868658577) -> (continuation of Deferred@1176515458 after retry RPC after error@915115750) -> (continuation of Deferred@1690781239 after retry RPC after error@1651229979) -> (continuation of Deferred@630216439 after retry RPC after error@1325797405) -> (continuation of Deferred@296620340 after retry RPC after error@768691463) -> (continuation of Deferred@173113764 after retry RPC after error@1974005259), errback=(continuation of Deferred@902302560 after retry RPC after error@405610190) -> (continuation of Deferred@278417616 after retry RPC after error@868658577) -> (continuation of Deferred@1176515458 after retry RPC after error@915115750) -> (continuation of Deferred@1690781239 after retry RPC after error@1651229979) -> (continuation of Deferred@630216439 after retry RPC after error@1325797405) -> (continuation of Deferred@296620340 after retry RPC after error@768691463) -> (continuation of Deferred@173113764 after retry RPC after error@1974005259)))
20:28:02.691 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:02.691 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:02.692 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:02.692 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:02.692 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:02.692 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:02.692 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:02.692 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:02.693 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:02.711 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:02.711 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:02.712 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:02.712 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:02.712 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:02.712 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:02.712 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:02.712 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:02.713 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:02.731 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:02.732 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:02.733 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:02.733 [INFO - pool-40-thread-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:02.733 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:02.736 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:02.736 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:02.736 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:02.737 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:02.750 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:02.751 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:02.751 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:02.751 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:02.751 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:02.751 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:02.751 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:02.751 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:02.752 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:02.770 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:02.771 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:02.771 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:02.771 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:02.771 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:02.771 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:02.771 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:02.771 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:02.772 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:02.811 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:02.811 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:02.811 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:02.811 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:02.811 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:02.811 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:02.811 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:02.812 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:02.812 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:02.830 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:02.831 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:02.831 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:02.831 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:02.831 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:02.831 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:02.831 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:02.831 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:02.832 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:02.931 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:02.932 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:02.932 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:02.932 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:02.932 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:02.932 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:02.932 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:02.932 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:02.933 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:02.933 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [43ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [43ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [46ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [47ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [47ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [241ms] refreshing cache from master, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [243ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [243ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@1626888922(state=PENDING, result=null, callback=(continuation of Deferred@1450735625 after retry RPC after error@833154642) -> (continuation of Deferred@934508243 after retry RPC after error@356504693) -> (continuation of Deferred@9588459 after retry RPC after error@1992227541) -> (continuation of Deferred@1511464767 after retry RPC after error@836822132) -> (continuation of Deferred@588710325 after retry RPC after error@587083600) -> (continuation of Deferred@1081221847 after retry RPC after error@1980214761) -> (continuation of Deferred@1829109813 after retry RPC after error@1045509738), errback=(continuation of Deferred@1450735625 after retry RPC after error@833154642) -> (continuation of Deferred@934508243 after retry RPC after error@356504693) -> (continuation of Deferred@9588459 after retry RPC after error@1992227541) -> (continuation of Deferred@1511464767 after retry RPC after error@836822132) -> (continuation of Deferred@588710325 after retry RPC after error@587083600) -> (continuation of Deferred@1081221847 after retry RPC after error@1980214761) -> (continuation of Deferred@1829109813 after retry RPC after error@1045509738)))
20:28:02.933 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [43ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [43ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [46ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [47ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [47ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [241ms] refreshing cache from master, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [243ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [243ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@1626888922(state=PENDING, result=null, callback=(continuation of Deferred@1450735625 after retry RPC after error@833154642) -> (continuation of Deferred@934508243 after retry RPC after error@356504693) -> (continuation of Deferred@9588459 after retry RPC after error@1992227541) -> (continuation of Deferred@1511464767 after retry RPC after error@836822132) -> (continuation of Deferred@588710325 after retry RPC after error@587083600) -> (continuation of Deferred@1081221847 after retry RPC after error@1980214761) -> (continuation of Deferred@1829109813 after retry RPC after error@1045509738), errback=(continuation of Deferred@1450735625 after retry RPC after error@833154642) -> (continuation of Deferred@934508243 after retry RPC after error@356504693) -> (continuation of Deferred@9588459 after retry RPC after error@1992227541) -> (continuation of Deferred@1511464767 after retry RPC after error@836822132) -> (continuation of Deferred@588710325 after retry RPC after error@587083600) -> (continuation of Deferred@1081221847 after retry RPC after error@1980214761) -> (continuation of Deferred@1829109813 after retry RPC after error@1045509738)))
20:28:02.950 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:02.951 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:02.951 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:02.951 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:02.951 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:02.951 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:02.951 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:02.951 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:02.952 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:02.970 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:02.970 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:02.971 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:02.971 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:02.971 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:02.971 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:02.971 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:02.971 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:02.972 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:02.991 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:02.991 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:02.992 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:02.992 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:02.992 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:02.992 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:02.992 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:02.992 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:02.992 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:03.010 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:03.011 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:03.011 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:03.011 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:03.011 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:03.012 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:03.012 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:03.012 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:03.012 [WARN - kudu-nio-7] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:03.030 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:03.031 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:03.031 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:03.031 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:03.031 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:03.031 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:03.032 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:03.032 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:03.032 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:03.032 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:03.032 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:03.033 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:03.070 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:03.071 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:03.071 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:03.071 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:03.071 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:03.071 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:03.071 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:03.071 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:03.072 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:03.151 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:03.151 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:03.151 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:03.152 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:03.152 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:03.152 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:03.152 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:03.152 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:03.152 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:03.191 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:03.191 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:03.192 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:03.192 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:03.192 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:03.192 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:03.192 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:03.192 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:03.192 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:03.193 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [83ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [241ms] refreshing cache from master, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [243ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@127556260(state=PENDING, result=null, callback=(continuation of Deferred@423314173 after retry RPC after error@252762214) -> (continuation of Deferred@44871524 after retry RPC after error@1471580911) -> (continuation of Deferred@349771342 after retry RPC after error@486752091) -> (continuation of Deferred@2050192400 after retry RPC after error@431056971) -> (continuation of Deferred@1166435925 after retry RPC after error@1106997337) -> (continuation of Deferred@545853314 after retry RPC after error@854116584) -> (continuation of Deferred@1143645230 after retry RPC after error@1581874506), errback=(continuation of Deferred@423314173 after retry RPC after error@252762214) -> (continuation of Deferred@44871524 after retry RPC after error@1471580911) -> (continuation of Deferred@349771342 after retry RPC after error@486752091) -> (continuation of Deferred@2050192400 after retry RPC after error@431056971) -> (continuation of Deferred@1166435925 after retry RPC after error@1106997337) -> (continuation of Deferred@545853314 after retry RPC after error@854116584) -> (continuation of Deferred@1143645230 after retry RPC after error@1581874506)))
20:28:03.193 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [83ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [241ms] refreshing cache from master, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [243ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@127556260(state=PENDING, result=null, callback=(continuation of Deferred@423314173 after retry RPC after error@252762214) -> (continuation of Deferred@44871524 after retry RPC after error@1471580911) -> (continuation of Deferred@349771342 after retry RPC after error@486752091) -> (continuation of Deferred@2050192400 after retry RPC after error@431056971) -> (continuation of Deferred@1166435925 after retry RPC after error@1106997337) -> (continuation of Deferred@545853314 after retry RPC after error@854116584) -> (continuation of Deferred@1143645230 after retry RPC after error@1581874506), errback=(continuation of Deferred@423314173 after retry RPC after error@252762214) -> (continuation of Deferred@44871524 after retry RPC after error@1471580911) -> (continuation of Deferred@349771342 after retry RPC after error@486752091) -> (continuation of Deferred@2050192400 after retry RPC after error@431056971) -> (continuation of Deferred@1166435925 after retry RPC after error@1106997337) -> (continuation of Deferred@545853314 after retry RPC after error@854116584) -> (continuation of Deferred@1143645230 after retry RPC after error@1581874506)))
20:28:03.210 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:03.210 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:03.211 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:03.211 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:03.211 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:03.211 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:03.211 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:03.211 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:03.212 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:03.231 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:03.231 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:03.232 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:03.232 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:03.232 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:03.232 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:03.232 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:03.232 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:03.232 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:03.251 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:03.251 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:03.251 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:03.251 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:03.251 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:03.251 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:03.251 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:03.252 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:03.252 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:03.271 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:03.271 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:03.271 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:03.271 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:03.271 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:03.272 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:03.272 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:03.272 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:03.273 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:03.291 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:03.291 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:03.291 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:03.291 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:03.292 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:03.292 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:03.292 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:03.292 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:03.292 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:03.311 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:03.311 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:03.311 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:03.311 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:03.311 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:03.311 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:03.311 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:03.311 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:03.312 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:03.351 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:03.351 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:03.351 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:03.351 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:03.351 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:03.351 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:03.352 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:03.352 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:03.352 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:03.371 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:03.371 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:03.371 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:03.371 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:03.371 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:03.371 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:03.372 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:03.372 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:03.372 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:03.411 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:03.411 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:03.411 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:03.411 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:03.411 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:03.411 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:03.411 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:03.412 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:03.412 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:03.413 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [63ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@333812067(state=PENDING, result=null, callback=(continuation of Deferred@1248849595 after retry RPC after error@929220868) -> (continuation of Deferred@441429006 after retry RPC after error@622745773) -> (continuation of Deferred@271671427 after retry RPC after error@825930088) -> (continuation of Deferred@1458721949 after retry RPC after error@25081310) -> (continuation of Deferred@441513994 after retry RPC after error@2122693593) -> (continuation of Deferred@974609960 after retry RPC after error@1312021745) -> (continuation of Deferred@1396632981 after retry RPC after error@3596120) -> (continuation of Deferred@13645672 after retry RPC after error@1356113879), errback=(continuation of Deferred@1248849595 after retry RPC after error@929220868) -> (continuation of Deferred@441429006 after retry RPC after error@622745773) -> (continuation of Deferred@271671427 after retry RPC after error@825930088) -> (continuation of Deferred@1458721949 after retry RPC after error@25081310) -> (continuation of Deferred@441513994 after retry RPC after error@2122693593) -> (continuation of Deferred@974609960 after retry RPC after error@1312021745) -> (continuation of Deferred@1396632981 after retry RPC after error@3596120) -> (continuation of Deferred@13645672 after retry RPC after error@1356113879)))
20:28:03.413 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [63ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@333812067(state=PENDING, result=null, callback=(continuation of Deferred@1248849595 after retry RPC after error@929220868) -> (continuation of Deferred@441429006 after retry RPC after error@622745773) -> (continuation of Deferred@271671427 after retry RPC after error@825930088) -> (continuation of Deferred@1458721949 after retry RPC after error@25081310) -> (continuation of Deferred@441513994 after retry RPC after error@2122693593) -> (continuation of Deferred@974609960 after retry RPC after error@1312021745) -> (continuation of Deferred@1396632981 after retry RPC after error@3596120) -> (continuation of Deferred@13645672 after retry RPC after error@1356113879), errback=(continuation of Deferred@1248849595 after retry RPC after error@929220868) -> (continuation of Deferred@441429006 after retry RPC after error@622745773) -> (continuation of Deferred@271671427 after retry RPC after error@825930088) -> (continuation of Deferred@1458721949 after retry RPC after error@25081310) -> (continuation of Deferred@441513994 after retry RPC after error@2122693593) -> (continuation of Deferred@974609960 after retry RPC after error@1312021745) -> (continuation of Deferred@1396632981 after retry RPC after error@3596120) -> (continuation of Deferred@13645672 after retry RPC after error@1356113879)))
20:28:03.431 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:03.431 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:03.431 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:03.431 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:03.431 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:03.431 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:03.431 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:03.431 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:03.432 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:03.450 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:03.451 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:03.451 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:03.451 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:03.451 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:03.451 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:03.451 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:03.451 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:03.452 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:03.470 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:03.471 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:03.471 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:03.471 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:03.471 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:03.471 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:03.471 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:03.471 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:03.472 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:03.490 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:03.491 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:03.491 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:03.491 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:03.491 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:03.491 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:03.491 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:03.491 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:03.492 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:03.510 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:03.511 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:03.511 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:03.511 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:03.511 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:03.511 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:03.511 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:03.511 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:03.512 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:03.550 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:03.551 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:03.551 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:03.551 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:03.551 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:03.551 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:03.551 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:03.551 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:03.552 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:03.611 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:03.611 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:03.611 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:03.611 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:03.611 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:03.611 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:03.611 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:03.611 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:03.612 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:03.612 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=181), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@1127379866(state=PENDING, result=null, callback=(continuation of Deferred@908955188 after retry RPC after error@1463728318) -> (continuation of Deferred@1457116032 after retry RPC after error@111662178) -> (continuation of Deferred@1545713324 after retry RPC after error@2130185950) -> (continuation of Deferred@1713739602 after retry RPC after error@1587104560) -> (continuation of Deferred@257688217 after retry RPC after error@829190994) -> (continuation of Deferred@2067397930 after retry RPC after error@1869478326), errback=(continuation of Deferred@908955188 after retry RPC after error@1463728318) -> (continuation of Deferred@1457116032 after retry RPC after error@111662178) -> (continuation of Deferred@1545713324 after retry RPC after error@2130185950) -> (continuation of Deferred@1713739602 after retry RPC after error@1587104560) -> (continuation of Deferred@257688217 after retry RPC after error@829190994) -> (continuation of Deferred@2067397930 after retry RPC after error@1869478326)))
20:28:03.612 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=181), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@1127379866(state=PENDING, result=null, callback=(continuation of Deferred@908955188 after retry RPC after error@1463728318) -> (continuation of Deferred@1457116032 after retry RPC after error@111662178) -> (continuation of Deferred@1545713324 after retry RPC after error@2130185950) -> (continuation of Deferred@1713739602 after retry RPC after error@1587104560) -> (continuation of Deferred@257688217 after retry RPC after error@829190994) -> (continuation of Deferred@2067397930 after retry RPC after error@1869478326), errback=(continuation of Deferred@908955188 after retry RPC after error@1463728318) -> (continuation of Deferred@1457116032 after retry RPC after error@111662178) -> (continuation of Deferred@1545713324 after retry RPC after error@2130185950) -> (continuation of Deferred@1713739602 after retry RPC after error@1587104560) -> (continuation of Deferred@257688217 after retry RPC after error@829190994) -> (continuation of Deferred@2067397930 after retry RPC after error@1869478326)))
20:28:03.630 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:03.631 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:03.631 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:03.631 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:03.631 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:03.631 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:03.631 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:03.631 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:03.632 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:03.650 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:03.651 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:03.651 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:03.651 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:03.651 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:03.651 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:03.651 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:03.651 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:03.652 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:03.670 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:03.671 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:03.671 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:03.671 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:03.671 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:03.671 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:03.671 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:03.671 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:03.672 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:03.690 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:03.691 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:03.691 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:03.691 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:03.691 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:03.691 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:03.691 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:03.691 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:03.692 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:03.710 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:03.711 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:03.711 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:03.711 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:03.711 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:03.711 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:03.711 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:03.711 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:03.712 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:03.751 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:03.751 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:03.751 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:03.751 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:03.751 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:03.751 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:03.751 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:03.751 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:03.752 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:03.831 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:03.831 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:03.832 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:03.832 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:03.832 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:03.832 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:03.832 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:03.832 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:03.832 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:03.832 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [201ms] refreshing cache from master, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@889401775(state=PENDING, result=null, callback=(continuation of Deferred@349251548 after retry RPC after error@583977208) -> (continuation of Deferred@325041572 after retry RPC after error@1065183273) -> (continuation of Deferred@571426781 after retry RPC after error@1993527916) -> (continuation of Deferred@403628943 after retry RPC after error@1290202247) -> (continuation of Deferred@336558953 after retry RPC after error@1949211190) -> (continuation of Deferred@466816423 after retry RPC after error@1613387918), errback=(continuation of Deferred@349251548 after retry RPC after error@583977208) -> (continuation of Deferred@325041572 after retry RPC after error@1065183273) -> (continuation of Deferred@571426781 after retry RPC after error@1993527916) -> (continuation of Deferred@403628943 after retry RPC after error@1290202247) -> (continuation of Deferred@336558953 after retry RPC after error@1949211190) -> (continuation of Deferred@466816423 after retry RPC after error@1613387918)))
20:28:03.835 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [201ms] refreshing cache from master, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@889401775(state=PENDING, result=null, callback=(continuation of Deferred@349251548 after retry RPC after error@583977208) -> (continuation of Deferred@325041572 after retry RPC after error@1065183273) -> (continuation of Deferred@571426781 after retry RPC after error@1993527916) -> (continuation of Deferred@403628943 after retry RPC after error@1290202247) -> (continuation of Deferred@336558953 after retry RPC after error@1949211190) -> (continuation of Deferred@466816423 after retry RPC after error@1613387918), errback=(continuation of Deferred@349251548 after retry RPC after error@583977208) -> (continuation of Deferred@325041572 after retry RPC after error@1065183273) -> (continuation of Deferred@571426781 after retry RPC after error@1993527916) -> (continuation of Deferred@403628943 after retry RPC after error@1290202247) -> (continuation of Deferred@336558953 after retry RPC after error@1949211190) -> (continuation of Deferred@466816423 after retry RPC after error@1613387918)))
20:28:03.851 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:03.851 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:03.852 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:03.852 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:03.852 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:03.852 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:03.852 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:03.852 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:03.852 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:03.871 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:03.871 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:03.872 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:03.872 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:03.872 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:03.872 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:03.872 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:03.873 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:03.873 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:03.891 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:03.891 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:03.891 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:03.891 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:03.891 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:03.892 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:03.892 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:03.892 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:03.892 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:03.911 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:03.911 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:03.911 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:03.911 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:03.911 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:03.911 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:03.912 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:03.912 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:03.912 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:03.931 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:03.931 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:03.931 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:03.931 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:03.931 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:03.931 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:03.931 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:03.931 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:03.932 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:03.950 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:03.951 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:03.951 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:03.951 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:03.951 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:03.951 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:03.951 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:03.951 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:03.952 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:03.970 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:03.971 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:03.971 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:03.971 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:03.971 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:03.971 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:03.971 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:03.971 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:03.972 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:04.091 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:04.091 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:04.092 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:04.092 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:04.092 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:04.092 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:04.092 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:04.092 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:04.093 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:04.093 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [19ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [39ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [60ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [80ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [99ms] refreshing cache from master, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [100ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [100ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [101ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [119ms] refreshing cache from master, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [120ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [120ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [121ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@1006799175(state=PENDING, result=null, callback=(continuation of Deferred@1084835559 after retry RPC after error@1392745183) -> (continuation of Deferred@337155593 after retry RPC after error@1022913916) -> (continuation of Deferred@921475706 after retry RPC after error@2046176026) -> (continuation of Deferred@697189107 after retry RPC after error@1449285939) -> (continuation of Deferred@68101839 after retry RPC after error@1290367919) -> (continuation of Deferred@388708635 after retry RPC after error@1405374732) -> (continuation of Deferred@780156133 after retry RPC after error@703809169), errback=(continuation of Deferred@1084835559 after retry RPC after error@1392745183) -> (continuation of Deferred@337155593 after retry RPC after error@1022913916) -> (continuation of Deferred@921475706 after retry RPC after error@2046176026) -> (continuation of Deferred@697189107 after retry RPC after error@1449285939) -> (continuation of Deferred@68101839 after retry RPC after error@1290367919) -> (continuation of Deferred@388708635 after retry RPC after error@1405374732) -> (continuation of Deferred@780156133 after retry RPC after error@703809169)))
20:28:04.094 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [19ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [39ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [60ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [80ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [99ms] refreshing cache from master, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [100ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [100ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [101ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [119ms] refreshing cache from master, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [120ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [120ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [121ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@1006799175(state=PENDING, result=null, callback=(continuation of Deferred@1084835559 after retry RPC after error@1392745183) -> (continuation of Deferred@337155593 after retry RPC after error@1022913916) -> (continuation of Deferred@921475706 after retry RPC after error@2046176026) -> (continuation of Deferred@697189107 after retry RPC after error@1449285939) -> (continuation of Deferred@68101839 after retry RPC after error@1290367919) -> (continuation of Deferred@388708635 after retry RPC after error@1405374732) -> (continuation of Deferred@780156133 after retry RPC after error@703809169), errback=(continuation of Deferred@1084835559 after retry RPC after error@1392745183) -> (continuation of Deferred@337155593 after retry RPC after error@1022913916) -> (continuation of Deferred@921475706 after retry RPC after error@2046176026) -> (continuation of Deferred@697189107 after retry RPC after error@1449285939) -> (continuation of Deferred@68101839 after retry RPC after error@1290367919) -> (continuation of Deferred@388708635 after retry RPC after error@1405374732) -> (continuation of Deferred@780156133 after retry RPC after error@703809169)))
20:28:04.111 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:04.111 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:04.112 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:04.112 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:04.112 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:04.112 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:04.112 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:04.112 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:04.113 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:04.131 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:04.131 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:04.132 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:04.132 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:04.132 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:04.132 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:04.132 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:04.132 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:04.132 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:04.151 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:04.152 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:04.152 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:04.152 [INFO - pool-40-thread-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:04.152 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:04.153 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:04.153 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:04.153 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:04.153 [WARN - kudu-nio-3] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:04.170 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:04.171 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:04.171 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:04.171 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:04.171 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:04.171 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:04.171 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:04.171 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:04.172 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:04.190 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:04.190 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:04.191 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:04.191 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:04.191 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:04.191 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:04.191 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:04.191 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:04.192 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:04.212 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:04.212 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:04.212 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:04.212 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:04.212 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:04.212 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:04.212 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:04.213 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:04.213 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:04.251 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:04.251 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:04.251 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:04.251 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:04.251 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:04.251 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:04.251 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:04.251 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:04.252 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:04.291 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:04.291 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:04.291 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:04.291 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:04.291 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:04.292 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:04.292 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:04.292 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:04.292 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:04.292 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=181), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [60ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [80ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [139ms] refreshing cache from master, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [140ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [140ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [141ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [179ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [181ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, deferred=Deferred@530506547(state=PENDING, result=null, callback=(continuation of Deferred@276180843 after retry RPC after error@1932769898) -> (continuation of Deferred@56818343 after retry RPC after error@1796015747) -> (continuation of Deferred@1252090366 after retry RPC after error@615473706) -> (continuation of Deferred@806093786 after retry RPC after error@867578438) -> (continuation of Deferred@1918438239 after retry RPC after error@758679820) -> (continuation of Deferred@2113559970 after retry RPC after error@2136765529) -> (continuation of Deferred@1172675358 after retry RPC after error@754715207), errback=(continuation of Deferred@276180843 after retry RPC after error@1932769898) -> (continuation of Deferred@56818343 after retry RPC after error@1796015747) -> (continuation of Deferred@1252090366 after retry RPC after error@615473706) -> (continuation of Deferred@806093786 after retry RPC after error@867578438) -> (continuation of Deferred@1918438239 after retry RPC after error@758679820) -> (continuation of Deferred@2113559970 after retry RPC after error@2136765529) -> (continuation of Deferred@1172675358 after retry RPC after error@754715207)))
20:28:04.293 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=181), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [60ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [80ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [139ms] refreshing cache from master, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [140ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [140ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [141ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [179ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [181ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, deferred=Deferred@530506547(state=PENDING, result=null, callback=(continuation of Deferred@276180843 after retry RPC after error@1932769898) -> (continuation of Deferred@56818343 after retry RPC after error@1796015747) -> (continuation of Deferred@1252090366 after retry RPC after error@615473706) -> (continuation of Deferred@806093786 after retry RPC after error@867578438) -> (continuation of Deferred@1918438239 after retry RPC after error@758679820) -> (continuation of Deferred@2113559970 after retry RPC after error@2136765529) -> (continuation of Deferred@1172675358 after retry RPC after error@754715207), errback=(continuation of Deferred@276180843 after retry RPC after error@1932769898) -> (continuation of Deferred@56818343 after retry RPC after error@1796015747) -> (continuation of Deferred@1252090366 after retry RPC after error@615473706) -> (continuation of Deferred@806093786 after retry RPC after error@867578438) -> (continuation of Deferred@1918438239 after retry RPC after error@758679820) -> (continuation of Deferred@2113559970 after retry RPC after error@2136765529) -> (continuation of Deferred@1172675358 after retry RPC after error@754715207)))
20:28:04.311 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:04.311 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:04.311 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:04.311 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:04.311 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:04.311 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:04.311 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:04.312 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:04.312 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:04.331 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:04.331 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:04.331 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:04.331 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:04.331 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:04.331 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:04.331 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:04.331 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:04.332 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:04.351 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:04.351 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:04.351 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:04.351 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:04.351 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:04.351 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:04.351 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:04.351 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:04.352 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:04.371 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:04.371 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:04.371 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:04.371 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:04.371 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:04.372 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:04.372 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:04.372 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:04.372 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:04.391 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:04.391 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:04.391 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:04.391 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:04.391 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:04.391 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:04.392 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:04.392 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:04.392 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:04.411 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:04.411 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:04.411 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:04.411 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:04.411 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:04.411 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:04.411 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:04.411 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:04.412 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:04.451 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:04.451 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:04.452 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:04.452 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:04.452 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:04.452 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:04.452 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:04.452 [INFO - pool-40-thread-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:04.453 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:04.550 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:04.550 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:04.551 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:04.551 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:04.551 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:04.551 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:04.551 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:04.551 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:04.552 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:04.552 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=241), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [140ms] refreshing cache from master, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [143ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@467453720(state=PENDING, result=null, callback=(continuation of Deferred@887658510 after retry RPC after error@1349510962) -> (continuation of Deferred@1816607215 after retry RPC after error@299812728) -> (continuation of Deferred@1636177317 after retry RPC after error@1325344976) -> (continuation of Deferred@463665313 after retry RPC after error@2082478506) -> (continuation of Deferred@769261750 after retry RPC after error@441713781) -> (continuation of Deferred@373260398 after retry RPC after error@1789802058) -> (continuation of Deferred@386679833 after retry RPC after error@759270689), errback=(continuation of Deferred@887658510 after retry RPC after error@1349510962) -> (continuation of Deferred@1816607215 after retry RPC after error@299812728) -> (continuation of Deferred@1636177317 after retry RPC after error@1325344976) -> (continuation of Deferred@463665313 after retry RPC after error@2082478506) -> (continuation of Deferred@769261750 after retry RPC after error@441713781) -> (continuation of Deferred@373260398 after retry RPC after error@1789802058) -> (continuation of Deferred@386679833 after retry RPC after error@759270689)))
20:28:04.552 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=241), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [140ms] refreshing cache from master, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [143ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@467453720(state=PENDING, result=null, callback=(continuation of Deferred@887658510 after retry RPC after error@1349510962) -> (continuation of Deferred@1816607215 after retry RPC after error@299812728) -> (continuation of Deferred@1636177317 after retry RPC after error@1325344976) -> (continuation of Deferred@463665313 after retry RPC after error@2082478506) -> (continuation of Deferred@769261750 after retry RPC after error@441713781) -> (continuation of Deferred@373260398 after retry RPC after error@1789802058) -> (continuation of Deferred@386679833 after retry RPC after error@759270689), errback=(continuation of Deferred@887658510 after retry RPC after error@1349510962) -> (continuation of Deferred@1816607215 after retry RPC after error@299812728) -> (continuation of Deferred@1636177317 after retry RPC after error@1325344976) -> (continuation of Deferred@463665313 after retry RPC after error@2082478506) -> (continuation of Deferred@769261750 after retry RPC after error@441713781) -> (continuation of Deferred@373260398 after retry RPC after error@1789802058) -> (continuation of Deferred@386679833 after retry RPC after error@759270689)))
20:28:04.571 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:04.571 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:04.572 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:04.572 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:04.572 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:04.572 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:04.572 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:04.572 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:04.573 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:04.591 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:04.591 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:04.591 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:04.591 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:04.591 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:04.592 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:04.592 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:04.592 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:04.592 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:04.611 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:04.611 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:04.611 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:04.611 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:04.611 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:04.612 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:04.612 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:04.612 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:04.612 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:04.631 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:04.631 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:04.631 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:04.631 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:04.631 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:04.631 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:04.631 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:04.631 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:04.632 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:04.650 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:04.651 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:04.651 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:04.651 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:04.651 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:04.651 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:04.651 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:04.651 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:04.652 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:04.670 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:04.671 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:04.671 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:04.671 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:04.671 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:04.671 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:04.671 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:04.671 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:04.672 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:04.710 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:04.711 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:04.711 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:04.711 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:04.711 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:04.711 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:04.711 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:04.711 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:04.712 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:04.811 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:04.811 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:04.811 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:04.811 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:04.811 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:04.812 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:04.812 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:04.812 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:04.812 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:04.812 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=241), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [20ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [39ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [40ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [60ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [80ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [99ms] refreshing cache from master, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [100ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [100ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [101ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [139ms] refreshing cache from master, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [140ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [140ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [141ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [239ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [240ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [241ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@1963622320(state=PENDING, result=null, callback=(continuation of Deferred@686926134 after retry RPC after error@1303606939) -> (continuation of Deferred@1146104003 after retry RPC after error@1652631558) -> (continuation of Deferred@1193051513 after retry RPC after error@298656943) -> (continuation of Deferred@1411256421 after retry RPC after error@1068811256) -> (continuation of Deferred@471907898 after retry RPC after error@1680873805) -> (continuation of Deferred@1074376854 after retry RPC after error@1497038956) -> (continuation of Deferred@1081324575 after retry RPC after error@611133229), errback=(continuation of Deferred@686926134 after retry RPC after error@1303606939) -> (continuation of Deferred@1146104003 after retry RPC after error@1652631558) -> (continuation of Deferred@1193051513 after retry RPC after error@298656943) -> (continuation of Deferred@1411256421 after retry RPC after error@1068811256) -> (continuation of Deferred@471907898 after retry RPC after error@1680873805) -> (continuation of Deferred@1074376854 after retry RPC after error@1497038956) -> (continuation of Deferred@1081324575 after retry RPC after error@611133229)))
20:28:04.813 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=241), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [20ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [39ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [40ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [60ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [80ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [99ms] refreshing cache from master, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [100ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [100ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [101ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [139ms] refreshing cache from master, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [140ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [140ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [141ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [239ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [240ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [241ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@1963622320(state=PENDING, result=null, callback=(continuation of Deferred@686926134 after retry RPC after error@1303606939) -> (continuation of Deferred@1146104003 after retry RPC after error@1652631558) -> (continuation of Deferred@1193051513 after retry RPC after error@298656943) -> (continuation of Deferred@1411256421 after retry RPC after error@1068811256) -> (continuation of Deferred@471907898 after retry RPC after error@1680873805) -> (continuation of Deferred@1074376854 after retry RPC after error@1497038956) -> (continuation of Deferred@1081324575 after retry RPC after error@611133229), errback=(continuation of Deferred@686926134 after retry RPC after error@1303606939) -> (continuation of Deferred@1146104003 after retry RPC after error@1652631558) -> (continuation of Deferred@1193051513 after retry RPC after error@298656943) -> (continuation of Deferred@1411256421 after retry RPC after error@1068811256) -> (continuation of Deferred@471907898 after retry RPC after error@1680873805) -> (continuation of Deferred@1074376854 after retry RPC after error@1497038956) -> (continuation of Deferred@1081324575 after retry RPC after error@611133229)))
20:28:04.831 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:04.831 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:04.831 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:04.831 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:04.831 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:04.831 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:04.831 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:04.832 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:04.832 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:04.850 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:04.851 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:04.851 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:04.851 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:04.851 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:04.851 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:04.851 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:04.851 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:04.852 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:04.870 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:04.871 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:04.871 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:04.871 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:04.871 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:04.871 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:04.871 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:04.871 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:04.872 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:04.890 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:04.891 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:04.891 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:04.891 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:04.891 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:04.891 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:04.891 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:04.891 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:04.892 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:04.910 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:04.911 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:04.911 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:04.911 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:04.911 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:04.911 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:04.911 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:04.911 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:04.912 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:04.930 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:04.931 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:04.931 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:04.931 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:04.931 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:04.931 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:04.931 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:04.931 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:04.932 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:04.991 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:04.991 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:04.991 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:04.991 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:04.991 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:04.992 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:04.992 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:04.992 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:04.992 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:05.091 [DEBUG - pool-40-thread-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=259), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@2092028008(state=PENDING, result=null, callback=(continuation of Deferred@2014114930 after retry RPC after error@1260628183) -> (continuation of Deferred@1583793784 after retry RPC after error@749109068) -> (continuation of Deferred@865513934 after retry RPC after error@1696977531) -> (continuation of Deferred@228101537 after retry RPC after error@316399425) -> (continuation of Deferred@348460053 after retry RPC after error@1597553404) -> (continuation of Deferred@377803071 after retry RPC after error@1525233017) -> (continuation of Deferred@1714485133 after retry RPC after error@2093305581), errback=(continuation of Deferred@2014114930 after retry RPC after error@1260628183) -> (continuation of Deferred@1583793784 after retry RPC after error@749109068) -> (continuation of Deferred@865513934 after retry RPC after error@1696977531) -> (continuation of Deferred@228101537 after retry RPC after error@316399425) -> (continuation of Deferred@348460053 after retry RPC after error@1597553404) -> (continuation of Deferred@377803071 after retry RPC after error@1525233017) -> (continuation of Deferred@1714485133 after retry RPC after error@2093305581)))
20:28:05.091 [WARN - pool-40-thread-1] (HashedWheelTimer.java:716) An exception was thrown by TimerTask.
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713)
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701)
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:787)
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501)
at java.base/java.lang.Thread.run(Thread.java:840)
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713)
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701)
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:787)
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501)
at java.base/java.lang.Thread.run(Thread.java:840)
20:28:05.111 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:05.111 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:05.112 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:05.112 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:05.112 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:05.112 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:05.112 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:05.112 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:05.112 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:05.131 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:05.131 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:05.132 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:05.132 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:05.132 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:05.132 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:05.132 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:05.132 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:05.132 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:05.151 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:05.151 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:05.152 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:05.152 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:05.152 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:05.152 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:05.152 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:05.152 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:05.153 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:05.171 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:05.172 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:05.172 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:05.172 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:05.172 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:05.172 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:05.172 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:05.172 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:05.173 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:05.190 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:05.191 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:05.191 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:05.191 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:05.191 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:05.191 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:05.191 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:05.192 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:05.192 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:05.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:28:05.197265 4132 master_runner.cc:186] Error getting master registration for 127.3.46.61:39407: Network error: Client connection negotiation failed: client connection to 127.3.46.61:39407: connect: Connection refused (error 111),
20:28:05.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:28:05.197470 4132 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
20:28:05.211 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:05.211 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:05.211 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:05.211 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:05.211 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:05.211 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:05.211 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:05.212 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:05.212 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:05.271 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:05.271 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:05.271 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:05.272 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:05.272 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:05.272 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:05.272 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:05.272 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:05.272 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:05.350 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:05.351 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:05.351 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:05.351 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:05.351 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:05.351 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:05.351 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:05.351 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:05.352 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:05.352 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=241), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [39ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [80ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [99ms] refreshing cache from master, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [100ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [100ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [101ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [161ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [239ms] refreshing cache from master, [239ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [239ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [239ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [240ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [240ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [241ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@1360725633(state=PENDING, result=null, callback=(continuation of Deferred@1702653547 after retry RPC after error@1200024790) -> (continuation of Deferred@1630742739 after retry RPC after error@1746208122) -> (continuation of Deferred@1855729957 after retry RPC after error@512528508) -> (continuation of Deferred@1711363790 after retry RPC after error@654287265) -> (continuation of Deferred@702916930 after retry RPC after error@1402000242) -> (continuation of Deferred@1070287080 after retry RPC after error@1692533785) -> (continuation of Deferred@1615404670 after retry RPC after error@899111494), errback=(continuation of Deferred@1702653547 after retry RPC after error@1200024790) -> (continuation of Deferred@1630742739 after retry RPC after error@1746208122) -> (continuation of Deferred@1855729957 after retry RPC after error@512528508) -> (continuation of Deferred@1711363790 after retry RPC after error@654287265) -> (continuation of Deferred@702916930 after retry RPC after error@1402000242) -> (continuation of Deferred@1070287080 after retry RPC after error@1692533785) -> (continuation of Deferred@1615404670 after retry RPC after error@899111494)))
20:28:05.353 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=241), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [39ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [80ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [99ms] refreshing cache from master, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [100ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [100ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [101ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [161ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [239ms] refreshing cache from master, [239ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [239ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [239ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [240ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [240ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [241ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@1360725633(state=PENDING, result=null, callback=(continuation of Deferred@1702653547 after retry RPC after error@1200024790) -> (continuation of Deferred@1630742739 after retry RPC after error@1746208122) -> (continuation of Deferred@1855729957 after retry RPC after error@512528508) -> (continuation of Deferred@1711363790 after retry RPC after error@654287265) -> (continuation of Deferred@702916930 after retry RPC after error@1402000242) -> (continuation of Deferred@1070287080 after retry RPC after error@1692533785) -> (continuation of Deferred@1615404670 after retry RPC after error@899111494), errback=(continuation of Deferred@1702653547 after retry RPC after error@1200024790) -> (continuation of Deferred@1630742739 after retry RPC after error@1746208122) -> (continuation of Deferred@1855729957 after retry RPC after error@512528508) -> (continuation of Deferred@1711363790 after retry RPC after error@654287265) -> (continuation of Deferred@702916930 after retry RPC after error@1402000242) -> (continuation of Deferred@1070287080 after retry RPC after error@1692533785) -> (continuation of Deferred@1615404670 after retry RPC after error@899111494)))
20:28:05.370 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:05.371 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:05.371 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:05.371 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:05.371 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:05.371 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:05.371 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:05.371 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:05.372 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:05.390 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:05.390 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:05.391 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:05.391 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:05.391 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:05.391 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:05.391 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:05.391 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:05.392 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:05.411 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:05.411 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:05.412 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:05.412 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:05.412 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:05.412 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:05.412 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:05.412 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:05.413 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:05.431 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:05.431 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:05.431 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:05.431 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:05.432 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:05.432 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:05.432 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:05.432 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:05.432 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:05.451 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:05.451 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:05.451 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:05.452 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:05.452 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:05.452 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:05.452 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:05.452 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:05.453 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:05.470 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:05.471 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:05.471 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:05.471 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:05.471 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:05.471 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:05.471 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:05.471 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:05.472 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:05.551 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:05.551 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:05.552 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:05.552 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:05.552 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:05.552 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:05.552 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:05.552 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:05.553 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:05.553 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [43ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [83ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [181ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@629695614(state=PENDING, result=null, callback=(continuation of Deferred@1204052525 after retry RPC after error@522248886) -> (continuation of Deferred@430297104 after retry RPC after error@2145975502) -> (continuation of Deferred@467217428 after retry RPC after error@66345099) -> (continuation of Deferred@879115701 after retry RPC after error@206616955) -> (continuation of Deferred@1589349737 after retry RPC after error@2141894197) -> (continuation of Deferred@2049222624 after retry RPC after error@1932939966), errback=(continuation of Deferred@1204052525 after retry RPC after error@522248886) -> (continuation of Deferred@430297104 after retry RPC after error@2145975502) -> (continuation of Deferred@467217428 after retry RPC after error@66345099) -> (continuation of Deferred@879115701 after retry RPC after error@206616955) -> (continuation of Deferred@1589349737 after retry RPC after error@2141894197) -> (continuation of Deferred@2049222624 after retry RPC after error@1932939966)))
20:28:05.553 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [43ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [83ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [181ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@629695614(state=PENDING, result=null, callback=(continuation of Deferred@1204052525 after retry RPC after error@522248886) -> (continuation of Deferred@430297104 after retry RPC after error@2145975502) -> (continuation of Deferred@467217428 after retry RPC after error@66345099) -> (continuation of Deferred@879115701 after retry RPC after error@206616955) -> (continuation of Deferred@1589349737 after retry RPC after error@2141894197) -> (continuation of Deferred@2049222624 after retry RPC after error@1932939966), errback=(continuation of Deferred@1204052525 after retry RPC after error@522248886) -> (continuation of Deferred@430297104 after retry RPC after error@2145975502) -> (continuation of Deferred@467217428 after retry RPC after error@66345099) -> (continuation of Deferred@879115701 after retry RPC after error@206616955) -> (continuation of Deferred@1589349737 after retry RPC after error@2141894197) -> (continuation of Deferred@2049222624 after retry RPC after error@1932939966)))
20:28:05.571 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:05.571 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:05.572 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:05.572 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:05.572 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:05.572 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:05.572 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:05.572 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:05.572 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:05.591 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:05.591 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:05.592 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:05.592 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:05.592 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:05.592 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:05.592 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:05.592 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:05.592 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:05.611 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:05.611 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:05.611 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:05.611 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:05.611 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:05.612 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:05.612 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:05.612 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:05.612 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:05.631 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:05.631 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:05.631 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:05.631 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:05.631 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:05.631 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:05.631 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:05.631 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:05.632 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:05.651 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:05.651 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:05.651 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:05.651 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:05.651 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:05.651 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:05.651 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:05.651 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:05.652 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:05.670 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:05.671 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:05.671 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:05.671 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:05.671 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:05.671 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:05.671 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:05.671 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:05.672 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:05.690 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:05.691 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:05.691 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:05.691 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:05.691 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:05.691 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:05.691 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:05.691 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:05.692 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:05.751 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:05.751 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:05.752 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:05.752 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:05.752 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:05.752 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:05.752 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:05.752 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:05.752 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:05.753 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=181), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [1ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [39ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [40ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [60ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [80ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [99ms] refreshing cache from master, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [100ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [100ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [101ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [119ms] refreshing cache from master, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [120ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [120ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [121ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [179ms] refreshing cache from master, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@370129651(state=PENDING, result=null, callback=(continuation of Deferred@1731769483 after retry RPC after error@1820886567) -> (continuation of Deferred@914864786 after retry RPC after error@1563711123) -> (continuation of Deferred@2139766098 after retry RPC after error@1461396794) -> (continuation of Deferred@1556301409 after retry RPC after error@591135652) -> (continuation of Deferred@234015292 after retry RPC after error@63839309) -> (continuation of Deferred@1165537819 after retry RPC after error@110798899) -> (continuation of Deferred@488776182 after retry RPC after error@1862105707), errback=(continuation of Deferred@1731769483 after retry RPC after error@1820886567) -> (continuation of Deferred@914864786 after retry RPC after error@1563711123) -> (continuation of Deferred@2139766098 after retry RPC after error@1461396794) -> (continuation of Deferred@1556301409 after retry RPC after error@591135652) -> (continuation of Deferred@234015292 after retry RPC after error@63839309) -> (continuation of Deferred@1165537819 after retry RPC after error@110798899) -> (continuation of Deferred@488776182 after retry RPC after error@1862105707)))
20:28:05.753 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=181), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [1ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [39ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [40ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [60ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [80ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [99ms] refreshing cache from master, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [100ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [100ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [101ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [119ms] refreshing cache from master, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [120ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [120ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [121ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [179ms] refreshing cache from master, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@370129651(state=PENDING, result=null, callback=(continuation of Deferred@1731769483 after retry RPC after error@1820886567) -> (continuation of Deferred@914864786 after retry RPC after error@1563711123) -> (continuation of Deferred@2139766098 after retry RPC after error@1461396794) -> (continuation of Deferred@1556301409 after retry RPC after error@591135652) -> (continuation of Deferred@234015292 after retry RPC after error@63839309) -> (continuation of Deferred@1165537819 after retry RPC after error@110798899) -> (continuation of Deferred@488776182 after retry RPC after error@1862105707), errback=(continuation of Deferred@1731769483 after retry RPC after error@1820886567) -> (continuation of Deferred@914864786 after retry RPC after error@1563711123) -> (continuation of Deferred@2139766098 after retry RPC after error@1461396794) -> (continuation of Deferred@1556301409 after retry RPC after error@591135652) -> (continuation of Deferred@234015292 after retry RPC after error@63839309) -> (continuation of Deferred@1165537819 after retry RPC after error@110798899) -> (continuation of Deferred@488776182 after retry RPC after error@1862105707)))
20:28:05.771 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:05.771 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:05.772 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:05.772 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:05.772 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:05.772 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:05.772 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:05.772 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:05.772 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:05.791 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:05.791 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:05.792 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:05.792 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:05.792 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:05.792 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:05.792 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:05.792 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:05.792 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:05.811 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:05.811 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:05.811 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:05.811 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:05.811 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:05.811 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:05.811 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:05.812 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:05.812 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:05.831 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:05.831 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:05.831 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:05.831 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:05.831 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:05.831 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:05.831 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:05.832 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:05.832 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:05.851 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:05.851 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:05.851 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:05.851 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:05.851 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:05.852 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:05.852 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:05.852 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:05.852 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:05.891 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:05.891 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:05.891 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:05.892 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:05.892 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:05.892 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:05.892 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:05.892 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:05.892 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:05.911 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:05.911 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:05.912 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:05.912 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:05.912 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:05.912 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:05.912 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:05.912 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:05.913 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:05.951 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:05.951 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:05.952 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:05.952 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:05.952 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:05.952 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:05.952 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:05.952 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:05.953 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:05.953 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [1ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [19ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [39ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [40ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [60ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [121ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@1970098724(state=PENDING, result=null, callback=(continuation of Deferred@540928702 after retry RPC after error@1955802321) -> (continuation of Deferred@2098786879 after retry RPC after error@1660642250) -> (continuation of Deferred@769396017 after retry RPC after error@1086605823) -> (continuation of Deferred@2054669010 after retry RPC after error@1135287141) -> (continuation of Deferred@240329636 after retry RPC after error@1824135200) -> (continuation of Deferred@673130039 after retry RPC after error@863268962) -> (continuation of Deferred@615328683 after retry RPC after error@409619462), errback=(continuation of Deferred@540928702 after retry RPC after error@1955802321) -> (continuation of Deferred@2098786879 after retry RPC after error@1660642250) -> (continuation of Deferred@769396017 after retry RPC after error@1086605823) -> (continuation of Deferred@2054669010 after retry RPC after error@1135287141) -> (continuation of Deferred@240329636 after retry RPC after error@1824135200) -> (continuation of Deferred@673130039 after retry RPC after error@863268962) -> (continuation of Deferred@615328683 after retry RPC after error@409619462)))
20:28:05.954 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [1ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [19ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [39ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [40ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [60ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [121ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@1970098724(state=PENDING, result=null, callback=(continuation of Deferred@540928702 after retry RPC after error@1955802321) -> (continuation of Deferred@2098786879 after retry RPC after error@1660642250) -> (continuation of Deferred@769396017 after retry RPC after error@1086605823) -> (continuation of Deferred@2054669010 after retry RPC after error@1135287141) -> (continuation of Deferred@240329636 after retry RPC after error@1824135200) -> (continuation of Deferred@673130039 after retry RPC after error@863268962) -> (continuation of Deferred@615328683 after retry RPC after error@409619462), errback=(continuation of Deferred@540928702 after retry RPC after error@1955802321) -> (continuation of Deferred@2098786879 after retry RPC after error@1660642250) -> (continuation of Deferred@769396017 after retry RPC after error@1086605823) -> (continuation of Deferred@2054669010 after retry RPC after error@1135287141) -> (continuation of Deferred@240329636 after retry RPC after error@1824135200) -> (continuation of Deferred@673130039 after retry RPC after error@863268962) -> (continuation of Deferred@615328683 after retry RPC after error@409619462)))
20:28:05.971 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:05.971 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:05.971 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:05.972 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:05.971 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:05.972 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:05.972 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:05.972 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:05.972 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:05.991 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:05.991 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:05.991 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:05.991 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:05.991 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:05.992 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:05.992 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:05.992 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:05.992 [WARN - kudu-nio-5] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:06.010 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:06.011 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:06.011 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:06.011 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:06.011 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:06.011 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:06.011 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:06.011 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:06.012 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:06.030 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:06.030 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:06.031 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:06.031 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:06.031 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:06.031 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:06.031 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:06.031 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:06.032 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:06.051 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:06.051 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:06.052 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:06.052 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:06.052 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:06.052 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:06.052 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:06.052 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:06.053 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:06.071 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:06.071 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:06.072 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:06.072 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:06.072 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:06.072 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:06.072 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:06.072 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:06.073 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:06.091 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:06.091 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:06.092 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:06.092 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:06.092 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:06.092 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:06.092 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:06.092 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:06.093 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:06.131 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:06.131 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:06.132 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:06.132 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:06.132 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:06.132 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:06.132 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:06.132 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:06.133 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:06.190 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:06.190 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:06.191 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:06.191 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:06.191 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:06.191 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:06.191 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:06.191 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:06.192 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:06.192 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=221), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [83ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [103ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [123ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [161ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [163ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [222ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@947978523(state=PENDING, result=null, callback=(continuation of Deferred@1916111352 after retry RPC after error@425384451) -> (continuation of Deferred@991661978 after retry RPC after error@195217467) -> (continuation of Deferred@1426516734 after retry RPC after error@449030303) -> (continuation of Deferred@2109969154 after retry RPC after error@1623027601) -> (continuation of Deferred@42874978 after retry RPC after error@84869116) -> (continuation of Deferred@1681143772 after retry RPC after error@1878920492) -> (continuation of Deferred@473986457 after retry RPC after error@1849923826) -> (continuation of Deferred@730650673 after retry RPC after error@872691457), errback=(continuation of Deferred@1916111352 after retry RPC after error@425384451) -> (continuation of Deferred@991661978 after retry RPC after error@195217467) -> (continuation of Deferred@1426516734 after retry RPC after error@449030303) -> (continuation of Deferred@2109969154 after retry RPC after error@1623027601) -> (continuation of Deferred@42874978 after retry RPC after error@84869116) -> (continuation of Deferred@1681143772 after retry RPC after error@1878920492) -> (continuation of Deferred@473986457 after retry RPC after error@1849923826) -> (continuation of Deferred@730650673 after retry RPC after error@872691457)))
20:28:06.193 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=221), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [83ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [103ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [123ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [161ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [163ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [222ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@947978523(state=PENDING, result=null, callback=(continuation of Deferred@1916111352 after retry RPC after error@425384451) -> (continuation of Deferred@991661978 after retry RPC after error@195217467) -> (continuation of Deferred@1426516734 after retry RPC after error@449030303) -> (continuation of Deferred@2109969154 after retry RPC after error@1623027601) -> (continuation of Deferred@42874978 after retry RPC after error@84869116) -> (continuation of Deferred@1681143772 after retry RPC after error@1878920492) -> (continuation of Deferred@473986457 after retry RPC after error@1849923826) -> (continuation of Deferred@730650673 after retry RPC after error@872691457), errback=(continuation of Deferred@1916111352 after retry RPC after error@425384451) -> (continuation of Deferred@991661978 after retry RPC after error@195217467) -> (continuation of Deferred@1426516734 after retry RPC after error@449030303) -> (continuation of Deferred@2109969154 after retry RPC after error@1623027601) -> (continuation of Deferred@42874978 after retry RPC after error@84869116) -> (continuation of Deferred@1681143772 after retry RPC after error@1878920492) -> (continuation of Deferred@473986457 after retry RPC after error@1849923826) -> (continuation of Deferred@730650673 after retry RPC after error@872691457)))
20:28:06.210 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:06.210 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:06.211 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:06.211 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:06.211 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:06.211 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:06.211 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:06.211 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:06.212 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:06.231 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:06.231 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:06.232 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:06.232 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:06.232 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:06.232 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:06.232 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:06.232 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:06.233 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:06.251 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:06.251 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:06.252 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:06.252 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:06.252 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:06.252 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:06.252 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:06.252 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:06.252 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:06.252 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:06.253 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:06.253 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:06.271 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:06.271 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:06.272 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:06.272 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:06.272 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:06.272 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:06.272 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:06.272 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:06.272 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:06.291 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:06.291 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:06.291 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:06.291 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:06.291 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:06.292 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:06.292 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:06.292 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:06.292 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:06.311 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:06.311 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:06.311 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:06.311 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:06.311 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:06.312 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:06.312 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:06.312 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:06.312 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:06.351 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:06.351 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:06.351 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:06.351 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:06.352 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:06.352 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:06.352 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:06.352 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:06.352 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:06.411 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:06.411 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:06.412 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:06.412 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:06.412 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:06.412 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:06.412 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:06.412 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:06.412 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:06.413 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [43ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [201ms] refreshing cache from master, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@290079766(state=PENDING, result=null, callback=(continuation of Deferred@1205104639 after retry RPC after error@1555423754) -> (continuation of Deferred@514439500 after retry RPC after error@1568683131) -> (continuation of Deferred@969872291 after retry RPC after error@1243153186) -> (continuation of Deferred@430551284 after retry RPC after error@1163291734) -> (continuation of Deferred@71646548 after retry RPC after error@113716973) -> (continuation of Deferred@1379729617 after retry RPC after error@620749340) -> (continuation of Deferred@159237194 after retry RPC after error@913449934), errback=(continuation of Deferred@1205104639 after retry RPC after error@1555423754) -> (continuation of Deferred@514439500 after retry RPC after error@1568683131) -> (continuation of Deferred@969872291 after retry RPC after error@1243153186) -> (continuation of Deferred@430551284 after retry RPC after error@1163291734) -> (continuation of Deferred@71646548 after retry RPC after error@113716973) -> (continuation of Deferred@1379729617 after retry RPC after error@620749340) -> (continuation of Deferred@159237194 after retry RPC after error@913449934)))
20:28:06.413 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [43ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [201ms] refreshing cache from master, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@290079766(state=PENDING, result=null, callback=(continuation of Deferred@1205104639 after retry RPC after error@1555423754) -> (continuation of Deferred@514439500 after retry RPC after error@1568683131) -> (continuation of Deferred@969872291 after retry RPC after error@1243153186) -> (continuation of Deferred@430551284 after retry RPC after error@1163291734) -> (continuation of Deferred@71646548 after retry RPC after error@113716973) -> (continuation of Deferred@1379729617 after retry RPC after error@620749340) -> (continuation of Deferred@159237194 after retry RPC after error@913449934), errback=(continuation of Deferred@1205104639 after retry RPC after error@1555423754) -> (continuation of Deferred@514439500 after retry RPC after error@1568683131) -> (continuation of Deferred@969872291 after retry RPC after error@1243153186) -> (continuation of Deferred@430551284 after retry RPC after error@1163291734) -> (continuation of Deferred@71646548 after retry RPC after error@113716973) -> (continuation of Deferred@1379729617 after retry RPC after error@620749340) -> (continuation of Deferred@159237194 after retry RPC after error@913449934)))
20:28:06.431 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:06.431 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:06.431 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:06.432 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:06.432 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:06.432 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:06.432 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:06.432 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:06.432 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:06.451 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:06.451 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:06.451 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:06.451 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:06.452 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:06.452 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:06.452 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:06.452 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:06.452 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:06.471 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:06.471 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:06.471 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:06.472 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:06.472 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:06.472 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:06.472 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:06.472 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:06.472 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:06.491 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:06.491 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:06.491 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:06.491 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:06.491 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:06.492 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:06.492 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:06.492 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:06.492 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:06.511 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:06.511 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:06.511 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:06.511 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:06.511 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:06.511 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:06.511 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:06.512 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:06.512 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:06.530 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:06.531 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:06.531 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:06.531 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:06.531 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:06.531 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:06.531 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:06.531 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:06.532 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:06.591 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:06.591 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:06.591 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:06.591 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:06.591 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:06.591 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:06.591 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:06.591 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:06.592 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:06.690 [DEBUG - pool-40-thread-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=259), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [1ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [60ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [80ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [99ms] refreshing cache from master, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [100ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [100ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [101ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [159ms] refreshing cache from master, [159ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [159ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [160ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [160ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [161ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@810257542(state=PENDING, result=null, callback=(continuation of Deferred@433510060 after retry RPC after error@893445396) -> (continuation of Deferred@837754300 after retry RPC after error@1563199675) -> (continuation of Deferred@1918912004 after retry RPC after error@1085961497) -> (continuation of Deferred@166822479 after retry RPC after error@807516826) -> (continuation of Deferred@1082235644 after retry RPC after error@1392524616) -> (continuation of Deferred@1449541277 after retry RPC after error@1798220705) -> (continuation of Deferred@657758786 after retry RPC after error@1963727249), errback=(continuation of Deferred@433510060 after retry RPC after error@893445396) -> (continuation of Deferred@837754300 after retry RPC after error@1563199675) -> (continuation of Deferred@1918912004 after retry RPC after error@1085961497) -> (continuation of Deferred@166822479 after retry RPC after error@807516826) -> (continuation of Deferred@1082235644 after retry RPC after error@1392524616) -> (continuation of Deferred@1449541277 after retry RPC after error@1798220705) -> (continuation of Deferred@657758786 after retry RPC after error@1963727249)))
20:28:06.691 [WARN - pool-40-thread-1] (HashedWheelTimer.java:716) An exception was thrown by TimerTask.
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713)
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701)
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:787)
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501)
at java.base/java.lang.Thread.run(Thread.java:840)
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713)
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701)
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:787)
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501)
at java.base/java.lang.Thread.run(Thread.java:840)
20:28:06.710 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:06.711 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:06.711 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:06.711 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:06.711 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:06.711 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:06.711 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:06.711 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:06.712 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:06.730 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:06.731 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:06.731 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:06.731 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:06.731 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:06.731 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:06.731 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:06.731 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:06.732 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:06.750 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:06.751 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:06.751 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:06.751 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:06.751 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:06.751 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:06.751 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:06.751 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:06.753 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:06.770 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:06.771 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:06.771 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:06.772 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:06.772 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:06.772 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:06.772 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:06.772 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:06.772 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:06.791 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:06.791 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:06.791 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:06.791 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:06.791 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:06.791 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:06.791 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:06.792 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:06.792 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:06.831 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:06.831 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:06.831 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:06.831 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:06.831 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:06.831 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:06.831 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:06.831 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:06.832 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:06.871 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:06.871 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:06.871 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:06.871 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:06.871 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:06.871 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:06.871 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:06.871 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:06.872 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:06.890 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:06.891 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:06.891 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:06.891 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:06.891 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:06.891 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:06.891 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:06.891 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:06.892 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:06.892 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@1054048082(state=PENDING, result=null, callback=(continuation of Deferred@335668455 after retry RPC after error@418014017) -> (continuation of Deferred@97173026 after retry RPC after error@541489190) -> (continuation of Deferred@997955093 after retry RPC after error@1891637840) -> (continuation of Deferred@548114884 after retry RPC after error@868174527) -> (continuation of Deferred@1410578915 after retry RPC after error@619789227) -> (continuation of Deferred@1479064384 after retry RPC after error@285993032) -> (continuation of Deferred@1478395691 after retry RPC after error@1965016032), errback=(continuation of Deferred@335668455 after retry RPC after error@418014017) -> (continuation of Deferred@97173026 after retry RPC after error@541489190) -> (continuation of Deferred@997955093 after retry RPC after error@1891637840) -> (continuation of Deferred@548114884 after retry RPC after error@868174527) -> (continuation of Deferred@1410578915 after retry RPC after error@619789227) -> (continuation of Deferred@1479064384 after retry RPC after error@285993032) -> (continuation of Deferred@1478395691 after retry RPC after error@1965016032)))
20:28:06.893 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@1054048082(state=PENDING, result=null, callback=(continuation of Deferred@335668455 after retry RPC after error@418014017) -> (continuation of Deferred@97173026 after retry RPC after error@541489190) -> (continuation of Deferred@997955093 after retry RPC after error@1891637840) -> (continuation of Deferred@548114884 after retry RPC after error@868174527) -> (continuation of Deferred@1410578915 after retry RPC after error@619789227) -> (continuation of Deferred@1479064384 after retry RPC after error@285993032) -> (continuation of Deferred@1478395691 after retry RPC after error@1965016032), errback=(continuation of Deferred@335668455 after retry RPC after error@418014017) -> (continuation of Deferred@97173026 after retry RPC after error@541489190) -> (continuation of Deferred@997955093 after retry RPC after error@1891637840) -> (continuation of Deferred@548114884 after retry RPC after error@868174527) -> (continuation of Deferred@1410578915 after retry RPC after error@619789227) -> (continuation of Deferred@1479064384 after retry RPC after error@285993032) -> (continuation of Deferred@1478395691 after retry RPC after error@1965016032)))
20:28:06.911 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:06.912 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:06.912 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:06.912 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:06.912 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:06.912 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:06.912 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:06.912 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:06.912 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:06.930 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:06.930 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:06.931 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:06.931 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:06.931 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:06.931 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:06.931 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:06.931 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:06.932 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:06.951 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:06.951 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:06.951 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:06.952 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:06.952 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:06.952 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:06.952 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:06.952 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:06.952 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:06.971 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:06.971 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:06.971 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:06.971 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:06.972 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:06.972 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:06.972 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:06.972 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:06.972 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:06.991 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:06.991 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:06.991 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:06.991 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:06.991 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:06.991 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:06.991 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:06.991 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:06.993 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:07.011 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:07.011 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:07.012 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:07.012 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:07.012 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:07.012 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:07.012 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:07.012 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:07.012 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:07.071 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:07.071 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:07.072 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:07.072 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:07.072 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:07.072 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:07.072 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:07.072 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:07.073 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:07.073 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [1ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [20ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [80ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [99ms] refreshing cache from master, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [101ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@24593920(state=PENDING, result=null, callback=(continuation of Deferred@1833879506 after retry RPC after error@1885775717) -> (continuation of Deferred@509022833 after retry RPC after error@623575293) -> (continuation of Deferred@1044394973 after retry RPC after error@2069427967) -> (continuation of Deferred@1572517759 after retry RPC after error@543910109) -> (continuation of Deferred@520776150 after retry RPC after error@601463750) -> (continuation of Deferred@735101325 after retry RPC after error@1350990250), errback=(continuation of Deferred@1833879506 after retry RPC after error@1885775717) -> (continuation of Deferred@509022833 after retry RPC after error@623575293) -> (continuation of Deferred@1044394973 after retry RPC after error@2069427967) -> (continuation of Deferred@1572517759 after retry RPC after error@543910109) -> (continuation of Deferred@520776150 after retry RPC after error@601463750) -> (continuation of Deferred@735101325 after retry RPC after error@1350990250)))
20:28:07.073 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [1ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [20ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [80ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [99ms] refreshing cache from master, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [101ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@24593920(state=PENDING, result=null, callback=(continuation of Deferred@1833879506 after retry RPC after error@1885775717) -> (continuation of Deferred@509022833 after retry RPC after error@623575293) -> (continuation of Deferred@1044394973 after retry RPC after error@2069427967) -> (continuation of Deferred@1572517759 after retry RPC after error@543910109) -> (continuation of Deferred@520776150 after retry RPC after error@601463750) -> (continuation of Deferred@735101325 after retry RPC after error@1350990250), errback=(continuation of Deferred@1833879506 after retry RPC after error@1885775717) -> (continuation of Deferred@509022833 after retry RPC after error@623575293) -> (continuation of Deferred@1044394973 after retry RPC after error@2069427967) -> (continuation of Deferred@1572517759 after retry RPC after error@543910109) -> (continuation of Deferred@520776150 after retry RPC after error@601463750) -> (continuation of Deferred@735101325 after retry RPC after error@1350990250)))
20:28:07.091 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:07.091 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:07.092 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:07.092 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:07.092 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:07.092 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:07.092 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:07.092 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:07.092 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:07.111 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:07.111 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:07.111 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:07.111 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:07.112 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:07.112 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:07.112 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:07.112 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:07.113 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:07.130 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:07.131 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:07.131 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:07.131 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:07.131 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:07.131 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:07.131 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:07.131 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:07.132 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:07.150 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:07.151 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:07.151 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:07.151 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:07.151 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:07.151 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:07.151 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:07.152 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:07.153 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:07.170 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:07.171 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:07.171 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:07.171 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:07.171 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:07.171 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:07.171 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:07.171 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:07.172 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:07.210 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:07.211 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:07.211 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:07.212 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:07.212 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:07.212 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:07.212 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:07.212 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:07.212 [WARN - kudu-nio-3] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:07.230 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:07.231 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:07.231 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:07.231 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:07.231 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:07.231 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:07.231 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:07.231 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:07.232 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:07.250 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:07.251 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:07.251 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:07.251 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:07.251 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:07.251 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:07.251 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:07.251 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:07.252 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:07.253 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=161), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [19ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [40ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [60ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [80ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [119ms] refreshing cache from master, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [121ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [139ms] refreshing cache from master, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [140ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [140ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [141ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [159ms] refreshing cache from master, [159ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [159ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [159ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [160ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [160ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [161ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@386045457(state=PENDING, result=null, callback=(continuation of Deferred@1881334082 after retry RPC after error@926976325) -> (continuation of Deferred@512635248 after retry RPC after error@499093166) -> (continuation of Deferred@2062570666 after retry RPC after error@1237970960) -> (continuation of Deferred@639847912 after retry RPC after error@1191734652) -> (continuation of Deferred@1067791411 after retry RPC after error@971643559) -> (continuation of Deferred@85609599 after retry RPC after error@25010733) -> (continuation of Deferred@401541789 after retry RPC after error@2117666277), errback=(continuation of Deferred@1881334082 after retry RPC after error@926976325) -> (continuation of Deferred@512635248 after retry RPC after error@499093166) -> (continuation of Deferred@2062570666 after retry RPC after error@1237970960) -> (continuation of Deferred@639847912 after retry RPC after error@1191734652) -> (continuation of Deferred@1067791411 after retry RPC after error@971643559) -> (continuation of Deferred@85609599 after retry RPC after error@25010733) -> (continuation of Deferred@401541789 after retry RPC after error@2117666277)))
20:28:07.253 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=161), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [19ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [40ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [60ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [80ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [119ms] refreshing cache from master, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [121ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [139ms] refreshing cache from master, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [140ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [140ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [141ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [159ms] refreshing cache from master, [159ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [159ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [159ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [160ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [160ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [161ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@386045457(state=PENDING, result=null, callback=(continuation of Deferred@1881334082 after retry RPC after error@926976325) -> (continuation of Deferred@512635248 after retry RPC after error@499093166) -> (continuation of Deferred@2062570666 after retry RPC after error@1237970960) -> (continuation of Deferred@639847912 after retry RPC after error@1191734652) -> (continuation of Deferred@1067791411 after retry RPC after error@971643559) -> (continuation of Deferred@85609599 after retry RPC after error@25010733) -> (continuation of Deferred@401541789 after retry RPC after error@2117666277), errback=(continuation of Deferred@1881334082 after retry RPC after error@926976325) -> (continuation of Deferred@512635248 after retry RPC after error@499093166) -> (continuation of Deferred@2062570666 after retry RPC after error@1237970960) -> (continuation of Deferred@639847912 after retry RPC after error@1191734652) -> (continuation of Deferred@1067791411 after retry RPC after error@971643559) -> (continuation of Deferred@85609599 after retry RPC after error@25010733) -> (continuation of Deferred@401541789 after retry RPC after error@2117666277)))
20:28:07.270 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:07.271 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:07.271 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:07.271 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:07.271 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:07.271 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:07.271 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:07.271 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:07.272 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:07.290 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:07.290 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:07.291 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:07.291 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:07.291 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:07.291 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:07.291 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:07.291 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:07.292 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:07.310 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:07.310 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:07.311 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:07.311 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:07.311 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:07.311 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:07.311 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:07.311 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:07.312 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:07.331 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:07.331 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:07.332 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:07.332 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:07.332 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:07.332 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:07.332 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:07.332 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:07.332 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:07.351 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:07.351 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:07.351 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:07.352 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:07.352 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:07.352 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:07.352 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:07.352 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:07.353 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:07.371 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:07.371 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:07.371 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:07.372 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:07.372 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:07.372 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:07.372 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:07.372 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:07.372 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:07.410 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:07.411 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:07.411 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:07.411 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:07.411 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:07.411 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:07.411 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:07.411 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:07.412 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:07.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:28:07.501345 3891 master_proxy_rpc.cc:177] Failed to determine new leader Master: Timed out: ConnectToClusterRpc(addrs: 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127, num_attempts: 97) passed its deadline: Not found: no leader found: ConnectToClusterRpc(addrs: 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127, num_attempts: 1)
20:28:07.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:28:07.506771 3891 meta_cache.cc:788] Timed out: LookupRpcById { tablet: 'f9ccfdb32e164e61bde68775baab7f9d', attempt: 2 } failed: LookupRpcById timed out after deadline expired: LookupRpcById { tablet: 'f9ccfdb32e164e61bde68775baab7f9d', attempt: 2 } passed its deadline: Timed out: ConnectToClusterRpc(addrs: 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127, num_attempts: 97) passed its deadline: Not found: no leader found: ConnectToClusterRpc(addrs: 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127, num_attempts: 1)
20:28:07.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:28:07.509050 3893 master_proxy_rpc.cc:203] Re-attempting LookupRpcById request to leader Master (127.3.46.62:46071)
20:28:07.511 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:07.511 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:07.512 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:07.512 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:07.512 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:07.512 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:07.512 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:07.512 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:07.512 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:07.512 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [241ms] refreshing cache from master, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@1964248152(state=PENDING, result=null, callback=(continuation of Deferred@1187730230 after retry RPC after error@1115152525) -> (continuation of Deferred@683491247 after retry RPC after error@2107543596) -> (continuation of Deferred@1641037386 after retry RPC after error@475978101) -> (continuation of Deferred@1566563606 after retry RPC after error@1414084961) -> (continuation of Deferred@873168844 after retry RPC after error@1584700745) -> (continuation of Deferred@2103467260 after retry RPC after error@1627381760) -> (continuation of Deferred@258279116 after retry RPC after error@1971408060), errback=(continuation of Deferred@1187730230 after retry RPC after error@1115152525) -> (continuation of Deferred@683491247 after retry RPC after error@2107543596) -> (continuation of Deferred@1641037386 after retry RPC after error@475978101) -> (continuation of Deferred@1566563606 after retry RPC after error@1414084961) -> (continuation of Deferred@873168844 after retry RPC after error@1584700745) -> (continuation of Deferred@2103467260 after retry RPC after error@1627381760) -> (continuation of Deferred@258279116 after retry RPC after error@1971408060)))
20:28:07.513 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [241ms] refreshing cache from master, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@1964248152(state=PENDING, result=null, callback=(continuation of Deferred@1187730230 after retry RPC after error@1115152525) -> (continuation of Deferred@683491247 after retry RPC after error@2107543596) -> (continuation of Deferred@1641037386 after retry RPC after error@475978101) -> (continuation of Deferred@1566563606 after retry RPC after error@1414084961) -> (continuation of Deferred@873168844 after retry RPC after error@1584700745) -> (continuation of Deferred@2103467260 after retry RPC after error@1627381760) -> (continuation of Deferred@258279116 after retry RPC after error@1971408060), errback=(continuation of Deferred@1187730230 after retry RPC after error@1115152525) -> (continuation of Deferred@683491247 after retry RPC after error@2107543596) -> (continuation of Deferred@1641037386 after retry RPC after error@475978101) -> (continuation of Deferred@1566563606 after retry RPC after error@1414084961) -> (continuation of Deferred@873168844 after retry RPC after error@1584700745) -> (continuation of Deferred@2103467260 after retry RPC after error@1627381760) -> (continuation of Deferred@258279116 after retry RPC after error@1971408060)))
20:28:07.531 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:07.531 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:07.532 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:07.532 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:07.532 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:07.532 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:07.532 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:07.532 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:07.532 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:07.551 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:07.551 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:07.552 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:07.552 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:07.552 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:07.552 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:07.552 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:07.552 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:07.552 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:07.571 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:07.571 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:07.572 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:07.572 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:07.572 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:07.572 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:07.572 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:07.572 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:07.572 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:07.591 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:07.591 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:07.591 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:07.592 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:07.592 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:07.592 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:07.592 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:07.592 [INFO - pool-40-thread-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:07.592 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:07.610 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:07.611 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:07.611 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:07.611 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:07.611 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:07.611 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:07.611 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:07.611 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:07.612 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:07.630 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:07.631 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:07.631 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:07.631 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:07.631 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:07.631 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:07.631 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:07.631 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:07.632 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:07.650 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:07.650 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:07.651 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:07.651 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:07.651 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:07.651 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:07.651 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:07.651 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:07.652 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:07.671 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:07.671 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:07.672 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:07.672 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:07.672 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:07.672 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:07.672 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:07.672 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:07.673 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:07.770 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:07.771 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:07.771 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:07.771 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:07.771 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:07.771 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:07.771 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:07.771 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:07.772 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:07.772 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=241), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [1ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [59ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [80ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [99ms] refreshing cache from master, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [100ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [100ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [101ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [119ms] refreshing cache from master, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [120ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [120ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [121ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [239ms] refreshing cache from master, [239ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [239ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [239ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [240ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [240ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [241ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, deferred=Deferred@172918261(state=PENDING, result=null, callback=(continuation of Deferred@1999647149 after retry RPC after error@1593281626) -> (continuation of Deferred@1796519073 after retry RPC after error@361493787) -> (continuation of Deferred@1665139550 after retry RPC after error@28825136) -> (continuation of Deferred@1338559259 after retry RPC after error@86101204) -> (continuation of Deferred@1721267677 after retry RPC after error@1503734120) -> (continuation of Deferred@1345888596 after retry RPC after error@1542750443) -> (continuation of Deferred@2120691925 after retry RPC after error@786283544) -> (continuation of Deferred@997886614 after retry RPC after error@1157019338), errback=(continuation of Deferred@1999647149 after retry RPC after error@1593281626) -> (continuation of Deferred@1796519073 after retry RPC after error@361493787) -> (continuation of Deferred@1665139550 after retry RPC after error@28825136) -> (continuation of Deferred@1338559259 after retry RPC after error@86101204) -> (continuation of Deferred@1721267677 after retry RPC after error@1503734120) -> (continuation of Deferred@1345888596 after retry RPC after error@1542750443) -> (continuation of Deferred@2120691925 after retry RPC after error@786283544) -> (continuation of Deferred@997886614 after retry RPC after error@1157019338)))
20:28:07.773 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=241), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [1ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [59ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [80ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [99ms] refreshing cache from master, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [100ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [100ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [101ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [119ms] refreshing cache from master, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [120ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [120ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [121ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [239ms] refreshing cache from master, [239ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [239ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [239ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [240ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [240ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [241ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, deferred=Deferred@172918261(state=PENDING, result=null, callback=(continuation of Deferred@1999647149 after retry RPC after error@1593281626) -> (continuation of Deferred@1796519073 after retry RPC after error@361493787) -> (continuation of Deferred@1665139550 after retry RPC after error@28825136) -> (continuation of Deferred@1338559259 after retry RPC after error@86101204) -> (continuation of Deferred@1721267677 after retry RPC after error@1503734120) -> (continuation of Deferred@1345888596 after retry RPC after error@1542750443) -> (continuation of Deferred@2120691925 after retry RPC after error@786283544) -> (continuation of Deferred@997886614 after retry RPC after error@1157019338), errback=(continuation of Deferred@1999647149 after retry RPC after error@1593281626) -> (continuation of Deferred@1796519073 after retry RPC after error@361493787) -> (continuation of Deferred@1665139550 after retry RPC after error@28825136) -> (continuation of Deferred@1338559259 after retry RPC after error@86101204) -> (continuation of Deferred@1721267677 after retry RPC after error@1503734120) -> (continuation of Deferred@1345888596 after retry RPC after error@1542750443) -> (continuation of Deferred@2120691925 after retry RPC after error@786283544) -> (continuation of Deferred@997886614 after retry RPC after error@1157019338)))
20:28:07.790 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:07.791 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:07.791 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:07.791 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:07.791 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:07.791 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:07.791 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:07.791 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:07.792 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:07.810 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:07.810 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:07.811 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:07.811 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:07.811 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:07.811 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:07.811 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:07.811 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:07.811 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:07.831 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:07.831 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:07.832 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:07.832 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:07.832 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:07.832 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:07.832 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:07.832 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:07.832 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:07.851 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:07.851 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:07.852 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:07.852 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:07.852 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:07.852 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:07.852 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:07.852 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:07.852 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:07.871 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:07.871 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:07.872 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:07.872 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:07.872 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:07.872 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:07.872 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:07.872 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:07.873 [WARN - kudu-nio-3] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:07.911 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:07.911 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:07.911 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:07.911 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:07.911 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:07.912 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:07.912 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:07.912 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:07.912 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:07.990 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:07.990 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:07.991 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:07.991 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:07.991 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:07.991 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:07.991 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:07.991 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:07.992 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:08.030 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:08.030 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:08.031 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:08.031 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:08.031 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:08.031 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:08.031 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:08.031 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:08.032 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:08.032 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=241), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@1530119586(state=PENDING, result=null, callback=(continuation of Deferred@1872326381 after retry RPC after error@1022407514) -> (continuation of Deferred@323641466 after retry RPC after error@1452390756) -> (continuation of Deferred@1926978397 after retry RPC after error@1258835007) -> (continuation of Deferred@193051796 after retry RPC after error@1342205465) -> (continuation of Deferred@2016007927 after retry RPC after error@1900610681) -> (continuation of Deferred@2007659480 after retry RPC after error@613006226) -> (continuation of Deferred@304177050 after retry RPC after error@728951659), errback=(continuation of Deferred@1872326381 after retry RPC after error@1022407514) -> (continuation of Deferred@323641466 after retry RPC after error@1452390756) -> (continuation of Deferred@1926978397 after retry RPC after error@1258835007) -> (continuation of Deferred@193051796 after retry RPC after error@1342205465) -> (continuation of Deferred@2016007927 after retry RPC after error@1900610681) -> (continuation of Deferred@2007659480 after retry RPC after error@613006226) -> (continuation of Deferred@304177050 after retry RPC after error@728951659)))
20:28:08.032 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=241), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@1530119586(state=PENDING, result=null, callback=(continuation of Deferred@1872326381 after retry RPC after error@1022407514) -> (continuation of Deferred@323641466 after retry RPC after error@1452390756) -> (continuation of Deferred@1926978397 after retry RPC after error@1258835007) -> (continuation of Deferred@193051796 after retry RPC after error@1342205465) -> (continuation of Deferred@2016007927 after retry RPC after error@1900610681) -> (continuation of Deferred@2007659480 after retry RPC after error@613006226) -> (continuation of Deferred@304177050 after retry RPC after error@728951659), errback=(continuation of Deferred@1872326381 after retry RPC after error@1022407514) -> (continuation of Deferred@323641466 after retry RPC after error@1452390756) -> (continuation of Deferred@1926978397 after retry RPC after error@1258835007) -> (continuation of Deferred@193051796 after retry RPC after error@1342205465) -> (continuation of Deferred@2016007927 after retry RPC after error@1900610681) -> (continuation of Deferred@2007659480 after retry RPC after error@613006226) -> (continuation of Deferred@304177050 after retry RPC after error@728951659)))
20:28:08.051 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:08.052 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:08.052 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:08.052 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:08.052 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:08.052 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:08.052 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:08.052 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:08.053 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:08.070 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:08.070 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:08.071 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:08.071 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:08.071 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:08.071 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:08.071 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:08.071 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:08.071 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:08.091 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:08.091 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:08.092 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:08.092 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:08.092 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:08.092 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:08.092 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:08.092 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:08.093 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:08.111 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:08.111 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:08.112 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:08.112 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:08.112 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:08.112 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:08.112 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:08.112 [INFO - pool-40-thread-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:08.113 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:08.131 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:08.131 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:08.132 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:08.132 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:08.132 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:08.132 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:08.132 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:08.132 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:08.132 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:08.171 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:08.171 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:08.172 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:08.172 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:08.172 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:08.172 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:08.172 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:08.172 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:08.172 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:08.230 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:08.230 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:08.231 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:08.231 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:08.231 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:08.231 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:08.231 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:08.231 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:08.231 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:08.270 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:08.271 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:08.271 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:08.271 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:08.271 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:08.271 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:08.271 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:08.271 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:08.272 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:08.272 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=220), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [20ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [20ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [121ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [179ms] refreshing cache from master, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [180ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [180ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [180ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [181ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [219ms] refreshing cache from master, [219ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [219ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [220ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [220ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [221ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@1580987848(state=PENDING, result=null, callback=(continuation of Deferred@1212338042 after retry RPC after error@1824160359) -> (continuation of Deferred@1483680691 after retry RPC after error@1501237443) -> (continuation of Deferred@119291273 after retry RPC after error@944379471) -> (continuation of Deferred@433688646 after retry RPC after error@413497014) -> (continuation of Deferred@2125721694 after retry RPC after error@371423657) -> (continuation of Deferred@1127059595 after retry RPC after error@1879989918) -> (continuation of Deferred@291402177 after retry RPC after error@473807986), errback=(continuation of Deferred@1212338042 after retry RPC after error@1824160359) -> (continuation of Deferred@1483680691 after retry RPC after error@1501237443) -> (continuation of Deferred@119291273 after retry RPC after error@944379471) -> (continuation of Deferred@433688646 after retry RPC after error@413497014) -> (continuation of Deferred@2125721694 after retry RPC after error@371423657) -> (continuation of Deferred@1127059595 after retry RPC after error@1879989918) -> (continuation of Deferred@291402177 after retry RPC after error@473807986)))
20:28:08.272 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=220), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [20ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [20ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [121ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [179ms] refreshing cache from master, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [180ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [180ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [180ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [181ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [219ms] refreshing cache from master, [219ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [219ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [220ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [220ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [221ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@1580987848(state=PENDING, result=null, callback=(continuation of Deferred@1212338042 after retry RPC after error@1824160359) -> (continuation of Deferred@1483680691 after retry RPC after error@1501237443) -> (continuation of Deferred@119291273 after retry RPC after error@944379471) -> (continuation of Deferred@433688646 after retry RPC after error@413497014) -> (continuation of Deferred@2125721694 after retry RPC after error@371423657) -> (continuation of Deferred@1127059595 after retry RPC after error@1879989918) -> (continuation of Deferred@291402177 after retry RPC after error@473807986), errback=(continuation of Deferred@1212338042 after retry RPC after error@1824160359) -> (continuation of Deferred@1483680691 after retry RPC after error@1501237443) -> (continuation of Deferred@119291273 after retry RPC after error@944379471) -> (continuation of Deferred@433688646 after retry RPC after error@413497014) -> (continuation of Deferred@2125721694 after retry RPC after error@371423657) -> (continuation of Deferred@1127059595 after retry RPC after error@1879989918) -> (continuation of Deferred@291402177 after retry RPC after error@473807986)))
20:28:08.290 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:08.291 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:08.292 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:08.292 [INFO - pool-40-thread-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:08.292 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:08.292 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:08.293 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:08.293 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:08.293 [WARN - kudu-nio-3] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:08.310 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:08.311 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:08.311 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:08.311 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:08.311 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:08.312 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:08.312 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:08.312 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:08.312 [WARN - kudu-nio-5] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:08.330 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:08.331 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:08.331 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:08.331 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:08.331 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:08.331 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:08.331 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:08.331 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:08.332 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:08.350 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:08.351 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:08.351 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:08.351 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:08.351 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:08.351 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:08.351 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:08.351 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:08.352 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:08.370 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:08.371 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:08.371 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:08.371 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:08.371 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:08.371 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:08.371 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:08.371 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:08.372 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:08.410 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:08.411 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:08.411 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:08.411 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:08.411 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:08.411 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:08.411 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:08.411 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:08.412 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:08.491 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:08.491 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:08.491 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:08.491 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:08.491 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:08.491 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:08.491 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:08.492 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:08.492 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:08.493 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [3ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [3ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [200ms] refreshing cache from master, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@1408316273(state=PENDING, result=null, callback=(continuation of Deferred@585026385 after retry RPC after error@1333624786) -> (continuation of Deferred@1660180040 after retry RPC after error@944020444) -> (continuation of Deferred@1821662150 after retry RPC after error@1211316841) -> (continuation of Deferred@137716529 after retry RPC after error@347308944) -> (continuation of Deferred@1513595051 after retry RPC after error@184953334) -> (continuation of Deferred@680128332 after retry RPC after error@548296398), errback=(continuation of Deferred@585026385 after retry RPC after error@1333624786) -> (continuation of Deferred@1660180040 after retry RPC after error@944020444) -> (continuation of Deferred@1821662150 after retry RPC after error@1211316841) -> (continuation of Deferred@137716529 after retry RPC after error@347308944) -> (continuation of Deferred@1513595051 after retry RPC after error@184953334) -> (continuation of Deferred@680128332 after retry RPC after error@548296398)))
20:28:08.493 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [3ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [3ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [200ms] refreshing cache from master, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@1408316273(state=PENDING, result=null, callback=(continuation of Deferred@585026385 after retry RPC after error@1333624786) -> (continuation of Deferred@1660180040 after retry RPC after error@944020444) -> (continuation of Deferred@1821662150 after retry RPC after error@1211316841) -> (continuation of Deferred@137716529 after retry RPC after error@347308944) -> (continuation of Deferred@1513595051 after retry RPC after error@184953334) -> (continuation of Deferred@680128332 after retry RPC after error@548296398), errback=(continuation of Deferred@585026385 after retry RPC after error@1333624786) -> (continuation of Deferred@1660180040 after retry RPC after error@944020444) -> (continuation of Deferred@1821662150 after retry RPC after error@1211316841) -> (continuation of Deferred@137716529 after retry RPC after error@347308944) -> (continuation of Deferred@1513595051 after retry RPC after error@184953334) -> (continuation of Deferred@680128332 after retry RPC after error@548296398)))
20:28:08.510 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:08.511 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:08.511 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:08.511 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:08.511 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:08.511 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:08.511 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:08.511 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:08.512 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:08.530 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:08.531 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:08.531 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:08.531 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:08.531 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:08.531 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:08.531 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:08.531 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:08.532 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:08.550 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:08.551 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:08.551 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:08.551 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:08.551 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:08.551 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:08.551 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:08.551 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:08.552 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:08.570 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:08.570 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:08.571 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:08.571 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:08.571 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:08.571 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:08.571 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:08.571 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:08.572 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:08.591 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:08.591 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:08.592 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:08.592 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:08.592 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:08.592 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:08.592 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:08.592 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:08.593 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:08.631 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:08.631 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:08.632 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:08.632 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:08.632 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:08.632 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:08.632 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:08.632 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:08.632 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:08.671 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:08.671 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:08.672 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:08.672 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:08.672 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:08.672 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:08.672 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:08.672 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:08.672 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:08.730 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:08.731 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:08.731 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:08.731 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:08.731 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:08.731 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:08.731 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:08.731 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:08.732 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:08.750 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:08.750 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:08.752 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:08.752 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:08.752 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:08.752 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:08.752 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:08.752 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:08.752 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:08.752 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=241), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [161ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [242ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@237595245(state=PENDING, result=null, callback=(continuation of Deferred@1607404134 after retry RPC after error@1229691332) -> (continuation of Deferred@81737702 after retry RPC after error@1096834834) -> (continuation of Deferred@1717301602 after retry RPC after error@1934348756) -> (continuation of Deferred@240564316 after retry RPC after error@1301147211) -> (continuation of Deferred@1952765561 after retry RPC after error@1502374992) -> (continuation of Deferred@1654893413 after retry RPC after error@1082064588) -> (continuation of Deferred@318451620 after retry RPC after error@674185031), errback=(continuation of Deferred@1607404134 after retry RPC after error@1229691332) -> (continuation of Deferred@81737702 after retry RPC after error@1096834834) -> (continuation of Deferred@1717301602 after retry RPC after error@1934348756) -> (continuation of Deferred@240564316 after retry RPC after error@1301147211) -> (continuation of Deferred@1952765561 after retry RPC after error@1502374992) -> (continuation of Deferred@1654893413 after retry RPC after error@1082064588) -> (continuation of Deferred@318451620 after retry RPC after error@674185031)))
20:28:08.753 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=241), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [161ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [242ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@237595245(state=PENDING, result=null, callback=(continuation of Deferred@1607404134 after retry RPC after error@1229691332) -> (continuation of Deferred@81737702 after retry RPC after error@1096834834) -> (continuation of Deferred@1717301602 after retry RPC after error@1934348756) -> (continuation of Deferred@240564316 after retry RPC after error@1301147211) -> (continuation of Deferred@1952765561 after retry RPC after error@1502374992) -> (continuation of Deferred@1654893413 after retry RPC after error@1082064588) -> (continuation of Deferred@318451620 after retry RPC after error@674185031), errback=(continuation of Deferred@1607404134 after retry RPC after error@1229691332) -> (continuation of Deferred@81737702 after retry RPC after error@1096834834) -> (continuation of Deferred@1717301602 after retry RPC after error@1934348756) -> (continuation of Deferred@240564316 after retry RPC after error@1301147211) -> (continuation of Deferred@1952765561 after retry RPC after error@1502374992) -> (continuation of Deferred@1654893413 after retry RPC after error@1082064588) -> (continuation of Deferred@318451620 after retry RPC after error@674185031)))
20:28:08.771 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:08.771 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:08.772 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:08.772 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:08.772 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:08.772 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:08.772 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:08.772 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:08.772 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:08.791 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:08.791 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:08.792 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:08.792 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:08.792 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:08.792 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:08.792 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:08.792 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:08.792 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:08.811 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:08.811 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:08.811 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:08.811 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:08.812 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:08.812 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:08.812 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:08.812 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:08.812 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:08.831 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:08.831 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:08.831 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:08.831 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:08.831 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:08.831 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:08.831 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:08.832 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:08.832 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:08.851 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:08.851 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:08.851 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:08.851 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:08.851 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:08.851 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:08.851 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:08.851 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:08.852 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:08.891 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:08.891 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:08.891 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:08.891 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:08.892 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:08.892 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:08.892 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:08.892 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:08.892 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:08.931 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:08.931 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:08.931 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:08.931 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:08.931 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:08.932 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:08.932 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:08.932 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:08.932 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:08.971 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:08.971 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:08.971 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:08.971 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:08.971 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:08.971 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:08.972 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:08.972 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:08.972 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:08.972 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=201), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [1ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [39ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [60ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [80ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [119ms] refreshing cache from master, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [121ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [159ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [161ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [201ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@872129809(state=PENDING, result=null, callback=(continuation of Deferred@1434904445 after retry RPC after error@1694874170) -> (continuation of Deferred@1018567105 after retry RPC after error@1384902763) -> (continuation of Deferred@810829896 after retry RPC after error@2026434578) -> (continuation of Deferred@1958516443 after retry RPC after error@2042181775) -> (continuation of Deferred@1799170552 after retry RPC after error@1414143213) -> (continuation of Deferred@1154499313 after retry RPC after error@1948088228) -> (continuation of Deferred@1986916918 after retry RPC after error@1917072738), errback=(continuation of Deferred@1434904445 after retry RPC after error@1694874170) -> (continuation of Deferred@1018567105 after retry RPC after error@1384902763) -> (continuation of Deferred@810829896 after retry RPC after error@2026434578) -> (continuation of Deferred@1958516443 after retry RPC after error@2042181775) -> (continuation of Deferred@1799170552 after retry RPC after error@1414143213) -> (continuation of Deferred@1154499313 after retry RPC after error@1948088228) -> (continuation of Deferred@1986916918 after retry RPC after error@1917072738)))
20:28:08.973 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=201), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [1ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [39ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [60ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [80ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [119ms] refreshing cache from master, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [121ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [159ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [161ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [201ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@872129809(state=PENDING, result=null, callback=(continuation of Deferred@1434904445 after retry RPC after error@1694874170) -> (continuation of Deferred@1018567105 after retry RPC after error@1384902763) -> (continuation of Deferred@810829896 after retry RPC after error@2026434578) -> (continuation of Deferred@1958516443 after retry RPC after error@2042181775) -> (continuation of Deferred@1799170552 after retry RPC after error@1414143213) -> (continuation of Deferred@1154499313 after retry RPC after error@1948088228) -> (continuation of Deferred@1986916918 after retry RPC after error@1917072738), errback=(continuation of Deferred@1434904445 after retry RPC after error@1694874170) -> (continuation of Deferred@1018567105 after retry RPC after error@1384902763) -> (continuation of Deferred@810829896 after retry RPC after error@2026434578) -> (continuation of Deferred@1958516443 after retry RPC after error@2042181775) -> (continuation of Deferred@1799170552 after retry RPC after error@1414143213) -> (continuation of Deferred@1154499313 after retry RPC after error@1948088228) -> (continuation of Deferred@1986916918 after retry RPC after error@1917072738)))
20:28:08.991 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:08.991 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:08.991 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:08.991 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:08.992 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:08.992 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:08.992 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:08.992 [INFO - pool-40-thread-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:08.992 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:09.010 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:09.011 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:09.011 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:09.011 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:09.011 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:09.011 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:09.012 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:09.012 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:09.012 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:09.031 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:09.031 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:09.032 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:09.032 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:09.032 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:09.032 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:09.032 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:09.032 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:09.032 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:09.050 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:09.051 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:09.051 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:09.051 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:09.051 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:09.051 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:09.051 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:09.051 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:09.052 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:09.070 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:09.071 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:09.071 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:09.071 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:09.071 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:09.071 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:09.071 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:09.071 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:09.072 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:09.090 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:09.091 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:09.091 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:09.091 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:09.091 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:09.091 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:09.091 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:09.091 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:09.092 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:09.131 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:09.131 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:09.131 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:09.131 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:09.131 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:09.131 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:09.131 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:09.132 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:09.133 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:09.170 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:09.171 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:09.171 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:09.171 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:09.171 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:09.171 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:09.171 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:09.171 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:09.172 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:09.172 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=181), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [143ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@24587930(state=PENDING, result=null, callback=(continuation of Deferred@1220687521 after retry RPC after error@874416084) -> (continuation of Deferred@390689454 after retry RPC after error@796040483) -> (continuation of Deferred@1985790290 after retry RPC after error@918345834) -> (continuation of Deferred@1790705681 after retry RPC after error@1360790496) -> (continuation of Deferred@1096425152 after retry RPC after error@301330697) -> (continuation of Deferred@457426862 after retry RPC after error@712098853) -> (continuation of Deferred@1003254709 after retry RPC after error@84914300), errback=(continuation of Deferred@1220687521 after retry RPC after error@874416084) -> (continuation of Deferred@390689454 after retry RPC after error@796040483) -> (continuation of Deferred@1985790290 after retry RPC after error@918345834) -> (continuation of Deferred@1790705681 after retry RPC after error@1360790496) -> (continuation of Deferred@1096425152 after retry RPC after error@301330697) -> (continuation of Deferred@457426862 after retry RPC after error@712098853) -> (continuation of Deferred@1003254709 after retry RPC after error@84914300)))
20:28:09.173 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=181), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [143ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@24587930(state=PENDING, result=null, callback=(continuation of Deferred@1220687521 after retry RPC after error@874416084) -> (continuation of Deferred@390689454 after retry RPC after error@796040483) -> (continuation of Deferred@1985790290 after retry RPC after error@918345834) -> (continuation of Deferred@1790705681 after retry RPC after error@1360790496) -> (continuation of Deferred@1096425152 after retry RPC after error@301330697) -> (continuation of Deferred@457426862 after retry RPC after error@712098853) -> (continuation of Deferred@1003254709 after retry RPC after error@84914300), errback=(continuation of Deferred@1220687521 after retry RPC after error@874416084) -> (continuation of Deferred@390689454 after retry RPC after error@796040483) -> (continuation of Deferred@1985790290 after retry RPC after error@918345834) -> (continuation of Deferred@1790705681 after retry RPC after error@1360790496) -> (continuation of Deferred@1096425152 after retry RPC after error@301330697) -> (continuation of Deferred@457426862 after retry RPC after error@712098853) -> (continuation of Deferred@1003254709 after retry RPC after error@84914300)))
20:28:09.190 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:09.191 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:09.191 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:09.191 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:09.191 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:09.191 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:09.191 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:09.191 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:09.192 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:09.210 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:09.211 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:09.211 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:09.211 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:09.211 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:09.211 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:09.211 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:09.211 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:09.212 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:09.230 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:09.231 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:09.231 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:09.231 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:09.231 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:09.231 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:09.231 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:09.231 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:09.232 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:09.250 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:09.250 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:09.251 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:09.251 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:09.251 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:09.251 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:09.251 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:09.251 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:09.252 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:09.271 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:09.271 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:09.272 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:09.272 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:09.272 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:09.272 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:09.272 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:09.272 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:09.272 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:09.311 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:09.311 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:09.312 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:09.312 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:09.312 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:09.312 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:09.312 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:09.312 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:09.313 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:09.370 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:09.371 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:09.371 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:09.371 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:09.371 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:09.371 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:09.371 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:09.371 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:09.372 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:09.410 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:09.411 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:09.411 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:09.411 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:09.411 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:09.411 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:09.411 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:09.411 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:09.412 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:09.412 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=221), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [123ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [222ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, deferred=Deferred@669317970(state=PENDING, result=null, callback=(continuation of Deferred@91134576 after retry RPC after error@865009539) -> (continuation of Deferred@1540967978 after retry RPC after error@1533476968) -> (continuation of Deferred@707505262 after retry RPC after error@1855794255) -> (continuation of Deferred@2133104552 after retry RPC after error@472104036) -> (continuation of Deferred@1805396818 after retry RPC after error@1826921786) -> (continuation of Deferred@925294297 after retry RPC after error@921142469) -> (continuation of Deferred@1009516528 after retry RPC after error@1953413122), errback=(continuation of Deferred@91134576 after retry RPC after error@865009539) -> (continuation of Deferred@1540967978 after retry RPC after error@1533476968) -> (continuation of Deferred@707505262 after retry RPC after error@1855794255) -> (continuation of Deferred@2133104552 after retry RPC after error@472104036) -> (continuation of Deferred@1805396818 after retry RPC after error@1826921786) -> (continuation of Deferred@925294297 after retry RPC after error@921142469) -> (continuation of Deferred@1009516528 after retry RPC after error@1953413122)))
20:28:09.412 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=221), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [123ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [222ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, deferred=Deferred@669317970(state=PENDING, result=null, callback=(continuation of Deferred@91134576 after retry RPC after error@865009539) -> (continuation of Deferred@1540967978 after retry RPC after error@1533476968) -> (continuation of Deferred@707505262 after retry RPC after error@1855794255) -> (continuation of Deferred@2133104552 after retry RPC after error@472104036) -> (continuation of Deferred@1805396818 after retry RPC after error@1826921786) -> (continuation of Deferred@925294297 after retry RPC after error@921142469) -> (continuation of Deferred@1009516528 after retry RPC after error@1953413122), errback=(continuation of Deferred@91134576 after retry RPC after error@865009539) -> (continuation of Deferred@1540967978 after retry RPC after error@1533476968) -> (continuation of Deferred@707505262 after retry RPC after error@1855794255) -> (continuation of Deferred@2133104552 after retry RPC after error@472104036) -> (continuation of Deferred@1805396818 after retry RPC after error@1826921786) -> (continuation of Deferred@925294297 after retry RPC after error@921142469) -> (continuation of Deferred@1009516528 after retry RPC after error@1953413122)))
20:28:09.430 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:09.431 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:09.431 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:09.431 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:09.431 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:09.431 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:09.431 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:09.431 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:09.432 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:09.450 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:09.451 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:09.451 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:09.451 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:09.451 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:09.451 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:09.451 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:09.451 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:09.452 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:09.470 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:09.471 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:09.471 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:09.471 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:09.471 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:09.471 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:09.471 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:09.471 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:09.472 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:09.490 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:09.491 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:09.491 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:09.491 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:09.491 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:09.491 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:09.491 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:09.491 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:09.492 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:09.510 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:09.511 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:09.511 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:09.511 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:09.511 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:09.511 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:09.511 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:09.511 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:09.511 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:09.550 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:09.551 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:09.551 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:09.551 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:09.551 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:09.551 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:09.551 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:09.551 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:09.552 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:09.591 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:09.591 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:09.591 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:09.591 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:09.592 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:09.592 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:09.592 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:09.592 [INFO - pool-40-thread-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:09.592 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:09.611 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:09.611 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:09.611 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:09.611 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:09.611 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:09.612 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:09.612 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:09.612 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:09.612 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:09.612 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@333446224(state=PENDING, result=null, callback=(continuation of Deferred@1290331478 after retry RPC after error@1301667707) -> (continuation of Deferred@890762974 after retry RPC after error@1497805136) -> (continuation of Deferred@1385115003 after retry RPC after error@822234710) -> (continuation of Deferred@992608174 after retry RPC after error@1468597674) -> (continuation of Deferred@2084970197 after retry RPC after error@1603641333) -> (continuation of Deferred@724950822 after retry RPC after error@881736639) -> (continuation of Deferred@2020748109 after retry RPC after error@191588737), errback=(continuation of Deferred@1290331478 after retry RPC after error@1301667707) -> (continuation of Deferred@890762974 after retry RPC after error@1497805136) -> (continuation of Deferred@1385115003 after retry RPC after error@822234710) -> (continuation of Deferred@992608174 after retry RPC after error@1468597674) -> (continuation of Deferred@2084970197 after retry RPC after error@1603641333) -> (continuation of Deferred@724950822 after retry RPC after error@881736639) -> (continuation of Deferred@2020748109 after retry RPC after error@191588737)))
20:28:09.613 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@333446224(state=PENDING, result=null, callback=(continuation of Deferred@1290331478 after retry RPC after error@1301667707) -> (continuation of Deferred@890762974 after retry RPC after error@1497805136) -> (continuation of Deferred@1385115003 after retry RPC after error@822234710) -> (continuation of Deferred@992608174 after retry RPC after error@1468597674) -> (continuation of Deferred@2084970197 after retry RPC after error@1603641333) -> (continuation of Deferred@724950822 after retry RPC after error@881736639) -> (continuation of Deferred@2020748109 after retry RPC after error@191588737), errback=(continuation of Deferred@1290331478 after retry RPC after error@1301667707) -> (continuation of Deferred@890762974 after retry RPC after error@1497805136) -> (continuation of Deferred@1385115003 after retry RPC after error@822234710) -> (continuation of Deferred@992608174 after retry RPC after error@1468597674) -> (continuation of Deferred@2084970197 after retry RPC after error@1603641333) -> (continuation of Deferred@724950822 after retry RPC after error@881736639) -> (continuation of Deferred@2020748109 after retry RPC after error@191588737)))
20:28:09.631 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:09.631 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:09.631 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:09.631 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:09.631 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:09.631 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:09.631 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:09.632 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:09.632 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:09.650 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:09.651 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:09.651 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:09.651 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:09.651 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:09.651 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:09.651 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:09.651 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:09.652 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:09.670 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:09.671 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:09.671 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:09.671 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:09.671 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:09.671 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:09.671 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:09.671 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:09.672 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:09.690 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:09.690 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:09.691 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:09.691 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:09.691 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:09.691 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:09.691 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:09.691 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:09.692 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:09.710 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:09.710 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:09.711 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:09.711 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:09.711 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:09.711 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:09.711 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:09.711 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:09.711 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:09.711 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:09.712 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:09.712 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:09.750 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:09.751 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:09.751 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:09.751 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:09.751 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:09.751 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:09.751 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:09.751 [INFO - pool-40-thread-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:09.752 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:09.810 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:09.811 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:09.811 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:09.811 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:09.811 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:09.811 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:09.811 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:09.811 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:09.812 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:09.830 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:09.830 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:09.831 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:09.831 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:09.831 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:09.831 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:09.831 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:09.831 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:09.832 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:09.832 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=201), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, deferred=Deferred@57193406(state=PENDING, result=null, callback=(continuation of Deferred@206930342 after retry RPC after error@1776317125) -> (continuation of Deferred@682028080 after retry RPC after error@1855279065) -> (continuation of Deferred@1315810076 after retry RPC after error@619814478) -> (continuation of Deferred@1000944012 after retry RPC after error@2139507367) -> (continuation of Deferred@1544567552 after retry RPC after error@1958464802) -> (continuation of Deferred@107788136 after retry RPC after error@1274152560) -> (continuation of Deferred@375909429 after retry RPC after error@1008726278), errback=(continuation of Deferred@206930342 after retry RPC after error@1776317125) -> (continuation of Deferred@682028080 after retry RPC after error@1855279065) -> (continuation of Deferred@1315810076 after retry RPC after error@619814478) -> (continuation of Deferred@1000944012 after retry RPC after error@2139507367) -> (continuation of Deferred@1544567552 after retry RPC after error@1958464802) -> (continuation of Deferred@107788136 after retry RPC after error@1274152560) -> (continuation of Deferred@375909429 after retry RPC after error@1008726278)))
20:28:09.832 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=201), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, deferred=Deferred@57193406(state=PENDING, result=null, callback=(continuation of Deferred@206930342 after retry RPC after error@1776317125) -> (continuation of Deferred@682028080 after retry RPC after error@1855279065) -> (continuation of Deferred@1315810076 after retry RPC after error@619814478) -> (continuation of Deferred@1000944012 after retry RPC after error@2139507367) -> (continuation of Deferred@1544567552 after retry RPC after error@1958464802) -> (continuation of Deferred@107788136 after retry RPC after error@1274152560) -> (continuation of Deferred@375909429 after retry RPC after error@1008726278), errback=(continuation of Deferred@206930342 after retry RPC after error@1776317125) -> (continuation of Deferred@682028080 after retry RPC after error@1855279065) -> (continuation of Deferred@1315810076 after retry RPC after error@619814478) -> (continuation of Deferred@1000944012 after retry RPC after error@2139507367) -> (continuation of Deferred@1544567552 after retry RPC after error@1958464802) -> (continuation of Deferred@107788136 after retry RPC after error@1274152560) -> (continuation of Deferred@375909429 after retry RPC after error@1008726278)))
20:28:09.851 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:09.851 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:09.852 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:09.852 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:09.852 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:09.852 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:09.852 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:09.852 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:09.852 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:09.871 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:09.871 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:09.872 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:09.872 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:09.872 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:09.872 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:09.872 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:09.872 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:09.872 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:09.891 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:09.891 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:09.891 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:09.891 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:09.891 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:09.891 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:09.892 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:09.892 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:09.892 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:09.911 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:09.911 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:09.911 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:09.911 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:09.911 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:09.911 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:09.912 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:09.912 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:09.912 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:09.930 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:09.931 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:09.931 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:09.931 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:09.931 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:09.932 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:09.933 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:09.933 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:09.933 [WARN - kudu-nio-3] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:09.971 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:09.971 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:09.971 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:09.971 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:09.971 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:09.971 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:09.971 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:09.971 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:09.972 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:10.011 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:10.011 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:10.012 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:10.012 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:10.012 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:10.012 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:10.012 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:10.012 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:10.012 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:10.012 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=161), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [39ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [60ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [80ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [119ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [120ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [120ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [121ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [159ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [161ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@756996334(state=PENDING, result=null, callback=(continuation of Deferred@1850379146 after retry RPC after error@1867516767) -> (continuation of Deferred@1740187006 after retry RPC after error@1018910927) -> (continuation of Deferred@102203913 after retry RPC after error@1777498983) -> (continuation of Deferred@1204691442 after retry RPC after error@567637220) -> (continuation of Deferred@459848989 after retry RPC after error@1631306836) -> (continuation of Deferred@1473030192 after retry RPC after error@1563726969), errback=(continuation of Deferred@1850379146 after retry RPC after error@1867516767) -> (continuation of Deferred@1740187006 after retry RPC after error@1018910927) -> (continuation of Deferred@102203913 after retry RPC after error@1777498983) -> (continuation of Deferred@1204691442 after retry RPC after error@567637220) -> (continuation of Deferred@459848989 after retry RPC after error@1631306836) -> (continuation of Deferred@1473030192 after retry RPC after error@1563726969)))
20:28:10.013 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=161), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [39ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [60ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [80ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [119ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [120ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [120ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [121ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [159ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [161ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@756996334(state=PENDING, result=null, callback=(continuation of Deferred@1850379146 after retry RPC after error@1867516767) -> (continuation of Deferred@1740187006 after retry RPC after error@1018910927) -> (continuation of Deferred@102203913 after retry RPC after error@1777498983) -> (continuation of Deferred@1204691442 after retry RPC after error@567637220) -> (continuation of Deferred@459848989 after retry RPC after error@1631306836) -> (continuation of Deferred@1473030192 after retry RPC after error@1563726969), errback=(continuation of Deferred@1850379146 after retry RPC after error@1867516767) -> (continuation of Deferred@1740187006 after retry RPC after error@1018910927) -> (continuation of Deferred@102203913 after retry RPC after error@1777498983) -> (continuation of Deferred@1204691442 after retry RPC after error@567637220) -> (continuation of Deferred@459848989 after retry RPC after error@1631306836) -> (continuation of Deferred@1473030192 after retry RPC after error@1563726969)))
20:28:10.031 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:10.031 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:10.032 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:10.032 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:10.032 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:10.032 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:10.032 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:10.032 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:10.033 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:10.050 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:10.051 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:10.051 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:10.051 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:10.051 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:10.051 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:10.051 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:10.051 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:10.052 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:10.070 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:10.071 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:10.071 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:10.071 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:10.071 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:10.071 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:10.071 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:10.071 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:10.072 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:10.090 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:10.091 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:10.091 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:10.091 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:10.091 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:10.091 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:10.091 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:10.091 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:10.092 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:10.110 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:10.111 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:10.111 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:10.111 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:10.111 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:10.111 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:10.112 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:10.112 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:10.114 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:10.150 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:10.151 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:10.151 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:10.151 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:10.151 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:10.151 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:10.151 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:10.151 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:10.152 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:10.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:28:10.201493 4132 master_runner.cc:186] Error getting master registration for 127.3.46.61:39407: Network error: Client connection negotiation failed: client connection to 127.3.46.61:39407: connect: Connection refused (error 111),
20:28:10.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:28:10.201910 4132 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
20:28:10.231 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:10.231 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:10.232 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:10.232 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:10.232 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:10.232 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:10.232 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:10.232 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:10.233 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:10.233 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [3ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [84ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [84ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [201ms] refreshing cache from master, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [203ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [203ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, deferred=Deferred@653430070(state=PENDING, result=null, callback=(continuation of Deferred@2069728341 after retry RPC after error@1763051304) -> (continuation of Deferred@1961571685 after retry RPC after error@1187159539) -> (continuation of Deferred@1374330458 after retry RPC after error@136113160) -> (continuation of Deferred@628207731 after retry RPC after error@2005834795) -> (continuation of Deferred@1913902286 after retry RPC after error@494930472) -> (continuation of Deferred@903529411 after retry RPC after error@909536919), errback=(continuation of Deferred@2069728341 after retry RPC after error@1763051304) -> (continuation of Deferred@1961571685 after retry RPC after error@1187159539) -> (continuation of Deferred@1374330458 after retry RPC after error@136113160) -> (continuation of Deferred@628207731 after retry RPC after error@2005834795) -> (continuation of Deferred@1913902286 after retry RPC after error@494930472) -> (continuation of Deferred@903529411 after retry RPC after error@909536919)))
20:28:10.234 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [3ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [84ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [84ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [201ms] refreshing cache from master, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [203ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [203ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, deferred=Deferred@653430070(state=PENDING, result=null, callback=(continuation of Deferred@2069728341 after retry RPC after error@1763051304) -> (continuation of Deferred@1961571685 after retry RPC after error@1187159539) -> (continuation of Deferred@1374330458 after retry RPC after error@136113160) -> (continuation of Deferred@628207731 after retry RPC after error@2005834795) -> (continuation of Deferred@1913902286 after retry RPC after error@494930472) -> (continuation of Deferred@903529411 after retry RPC after error@909536919), errback=(continuation of Deferred@2069728341 after retry RPC after error@1763051304) -> (continuation of Deferred@1961571685 after retry RPC after error@1187159539) -> (continuation of Deferred@1374330458 after retry RPC after error@136113160) -> (continuation of Deferred@628207731 after retry RPC after error@2005834795) -> (continuation of Deferred@1913902286 after retry RPC after error@494930472) -> (continuation of Deferred@903529411 after retry RPC after error@909536919)))
20:28:10.251 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:10.251 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:10.252 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:10.252 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:10.252 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:10.252 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:10.252 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:10.252 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:10.252 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:10.271 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:10.271 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:10.272 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:10.272 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:10.272 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:10.272 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:10.272 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:10.272 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:10.273 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:10.291 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:10.291 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:10.292 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:10.292 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:10.292 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:10.292 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:10.292 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:10.292 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:10.293 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:10.311 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:10.311 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:10.312 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:10.312 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:10.312 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:10.312 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:10.312 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:10.312 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:10.312 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:10.331 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:10.331 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:10.331 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:10.332 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:10.332 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:10.332 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:10.332 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:10.332 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:10.333 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:10.371 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:10.371 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:10.371 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:10.371 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:10.371 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:10.372 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:10.372 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:10.372 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:10.372 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:10.431 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:10.431 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:10.432 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:10.432 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:10.432 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:10.432 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:10.432 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:10.432 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:10.432 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:10.433 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=181), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [1ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [121ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@4521355(state=PENDING, result=null, callback=(continuation of Deferred@1484214222 after retry RPC after error@1535580566) -> (continuation of Deferred@1636767465 after retry RPC after error@1174407158) -> (continuation of Deferred@611002823 after retry RPC after error@1204019061) -> (continuation of Deferred@1583526202 after retry RPC after error@412820151) -> (continuation of Deferred@1698895218 after retry RPC after error@1008879320) -> (continuation of Deferred@1996675847 after retry RPC after error@911065546), errback=(continuation of Deferred@1484214222 after retry RPC after error@1535580566) -> (continuation of Deferred@1636767465 after retry RPC after error@1174407158) -> (continuation of Deferred@611002823 after retry RPC after error@1204019061) -> (continuation of Deferred@1583526202 after retry RPC after error@412820151) -> (continuation of Deferred@1698895218 after retry RPC after error@1008879320) -> (continuation of Deferred@1996675847 after retry RPC after error@911065546)))
20:28:10.433 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=181), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [1ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [121ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@4521355(state=PENDING, result=null, callback=(continuation of Deferred@1484214222 after retry RPC after error@1535580566) -> (continuation of Deferred@1636767465 after retry RPC after error@1174407158) -> (continuation of Deferred@611002823 after retry RPC after error@1204019061) -> (continuation of Deferred@1583526202 after retry RPC after error@412820151) -> (continuation of Deferred@1698895218 after retry RPC after error@1008879320) -> (continuation of Deferred@1996675847 after retry RPC after error@911065546), errback=(continuation of Deferred@1484214222 after retry RPC after error@1535580566) -> (continuation of Deferred@1636767465 after retry RPC after error@1174407158) -> (continuation of Deferred@611002823 after retry RPC after error@1204019061) -> (continuation of Deferred@1583526202 after retry RPC after error@412820151) -> (continuation of Deferred@1698895218 after retry RPC after error@1008879320) -> (continuation of Deferred@1996675847 after retry RPC after error@911065546)))
20:28:10.451 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:10.451 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:10.452 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:10.452 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:10.452 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:10.452 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:10.452 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:10.452 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:10.452 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:10.471 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:10.471 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:10.471 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:10.471 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:10.472 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:10.472 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:10.472 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:10.472 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:10.472 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:10.491 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:10.491 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:10.491 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:10.491 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:10.491 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:10.491 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:10.491 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:10.491 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:10.492 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:10.511 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:10.511 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:10.511 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:10.511 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:10.511 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:10.511 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:10.511 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:10.511 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:10.512 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:10.530 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:10.531 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:10.531 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:10.531 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:10.531 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:10.531 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:10.531 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:10.531 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:10.532 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:10.571 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:10.571 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:10.571 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:10.571 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:10.571 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:10.571 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:10.571 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:10.571 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:10.573 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:10.631 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:10.631 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:10.631 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:10.631 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:10.631 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:10.631 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:10.631 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:10.632 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:10.632 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:10.650 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:10.651 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:10.651 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:10.651 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:10.651 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:10.651 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:10.651 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:10.651 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:10.652 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:10.653 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=201), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [40ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [60ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [80ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [119ms] refreshing cache from master, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [120ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [120ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [179ms] refreshing cache from master, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [180ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [180ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [181ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [199ms] refreshing cache from master, [199ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [199ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [200ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [200ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [201ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@1793733121(state=PENDING, result=null, callback=(continuation of Deferred@542130587 after retry RPC after error@1754982158) -> (continuation of Deferred@310895213 after retry RPC after error@700882950) -> (continuation of Deferred@188244028 after retry RPC after error@883246274) -> (continuation of Deferred@421285041 after retry RPC after error@96864516) -> (continuation of Deferred@1303879680 after retry RPC after error@1403678240) -> (continuation of Deferred@682154272 after retry RPC after error@496204568) -> (continuation of Deferred@1834254090 after retry RPC after error@1838143470), errback=(continuation of Deferred@542130587 after retry RPC after error@1754982158) -> (continuation of Deferred@310895213 after retry RPC after error@700882950) -> (continuation of Deferred@188244028 after retry RPC after error@883246274) -> (continuation of Deferred@421285041 after retry RPC after error@96864516) -> (continuation of Deferred@1303879680 after retry RPC after error@1403678240) -> (continuation of Deferred@682154272 after retry RPC after error@496204568) -> (continuation of Deferred@1834254090 after retry RPC after error@1838143470)))
20:28:10.653 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=201), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [40ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [60ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [80ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [119ms] refreshing cache from master, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [120ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [120ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [179ms] refreshing cache from master, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [180ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [180ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [181ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [199ms] refreshing cache from master, [199ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [199ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [200ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [200ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [201ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@1793733121(state=PENDING, result=null, callback=(continuation of Deferred@542130587 after retry RPC after error@1754982158) -> (continuation of Deferred@310895213 after retry RPC after error@700882950) -> (continuation of Deferred@188244028 after retry RPC after error@883246274) -> (continuation of Deferred@421285041 after retry RPC after error@96864516) -> (continuation of Deferred@1303879680 after retry RPC after error@1403678240) -> (continuation of Deferred@682154272 after retry RPC after error@496204568) -> (continuation of Deferred@1834254090 after retry RPC after error@1838143470), errback=(continuation of Deferred@542130587 after retry RPC after error@1754982158) -> (continuation of Deferred@310895213 after retry RPC after error@700882950) -> (continuation of Deferred@188244028 after retry RPC after error@883246274) -> (continuation of Deferred@421285041 after retry RPC after error@96864516) -> (continuation of Deferred@1303879680 after retry RPC after error@1403678240) -> (continuation of Deferred@682154272 after retry RPC after error@496204568) -> (continuation of Deferred@1834254090 after retry RPC after error@1838143470)))
20:28:10.670 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:10.671 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:10.671 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:10.671 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:10.671 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:10.671 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:10.671 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:10.671 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:10.672 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:10.690 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:10.690 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:10.691 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:10.691 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:10.691 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:10.691 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:10.691 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:10.691 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:10.692 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:10.711 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:10.711 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:10.712 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:10.712 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:10.712 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:10.712 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:10.712 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:10.712 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:10.713 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:10.731 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:10.731 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:10.731 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:10.732 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:10.732 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:10.732 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:10.732 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:10.732 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:10.732 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:10.751 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:10.751 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:10.751 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:10.752 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:10.752 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:10.752 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:10.752 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:10.752 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:10.752 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:10.791 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:10.791 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:10.792 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:10.792 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:10.792 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:10.792 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:10.792 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:10.792 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:10.792 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:10.831 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:10.831 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:10.832 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:10.832 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:10.832 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:10.832 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:10.832 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:10.832 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:10.832 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:10.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:28:10.856195 4450 raft_consensus.cc:493] T 00000000000000000000000000000000 P 0db1d539946942f794b0c4ec81f8f4b4 [term 4 FOLLOWER]: Starting pre-election (detected failure of leader b1731f8308c145f686f88efd6a54b851)
20:28:10.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:28:10.856432 4450 raft_consensus.cc:515] T 00000000000000000000000000000000 P 0db1d539946942f794b0c4ec81f8f4b4 [term 4 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0db1d539946942f794b0c4ec81f8f4b4" member_type: VOTER last_known_addr { host: "127.3.46.62" port: 46071 } } peers { permanent_uuid: "d6c270b387b04f17b2a58bfcb0eb4f82" member_type: VOTER last_known_addr { host: "127.3.46.61" port: 39407 } } peers { permanent_uuid: "b1731f8308c145f686f88efd6a54b851" member_type: VOTER last_known_addr { host: "127.3.46.60" port: 36127 } }
20:28:10.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:28:10.857196 4450 leader_election.cc:290] T 00000000000000000000000000000000 P 0db1d539946942f794b0c4ec81f8f4b4 [CANDIDATE]: Term 5 pre-election: Requested pre-vote from peers d6c270b387b04f17b2a58bfcb0eb4f82 (127.3.46.61:39407), b1731f8308c145f686f88efd6a54b851 (127.3.46.60:36127)
20:28:10.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:28:10.858441 4149 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.3.46.61:39407: connect: Connection refused (error 111) [suppressed 3 similar messages]
20:28:10.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:28:10.859822 4149 leader_election.cc:336] T 00000000000000000000000000000000 P 0db1d539946942f794b0c4ec81f8f4b4 [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer d6c270b387b04f17b2a58bfcb0eb4f82 (127.3.46.61:39407): Network error: Client connection negotiation failed: client connection to 127.3.46.61:39407: connect: Connection refused (error 111)
20:28:10.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:28:10.860033 4148 leader_election.cc:336] T 00000000000000000000000000000000 P 0db1d539946942f794b0c4ec81f8f4b4 [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer b1731f8308c145f686f88efd6a54b851 (127.3.46.60:36127): Network error: Client connection negotiation failed: client connection to 127.3.46.60:36127: connect: Connection refused (error 111)
20:28:10.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:28:10.860142 4148 leader_election.cc:304] T 00000000000000000000000000000000 P 0db1d539946942f794b0c4ec81f8f4b4 [CANDIDATE]: Term 5 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 0db1d539946942f794b0c4ec81f8f4b4; no voters: b1731f8308c145f686f88efd6a54b851, d6c270b387b04f17b2a58bfcb0eb4f82
20:28:10.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:28:10.860375 4450 raft_consensus.cc:2749] T 00000000000000000000000000000000 P 0db1d539946942f794b0c4ec81f8f4b4 [term 4 FOLLOWER]: Leader pre-election lost for term 5. Reason: could not achieve majority
20:28:10.890 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:10.890 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:10.891 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:10.891 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:10.891 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:10.891 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:10.891 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:10.891 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:10.891 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:10.892 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=221), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [43ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [161ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@833531004(state=PENDING, result=null, callback=(continuation of Deferred@37602281 after retry RPC after error@1056147148) -> (continuation of Deferred@677724785 after retry RPC after error@1827460685) -> (continuation of Deferred@10642098 after retry RPC after error@574968204) -> (continuation of Deferred@548534655 after retry RPC after error@711269786) -> (continuation of Deferred@960581014 after retry RPC after error@822129155) -> (continuation of Deferred@517332829 after retry RPC after error@484350160) -> (continuation of Deferred@801135392 after retry RPC after error@467956849), errback=(continuation of Deferred@37602281 after retry RPC after error@1056147148) -> (continuation of Deferred@677724785 after retry RPC after error@1827460685) -> (continuation of Deferred@10642098 after retry RPC after error@574968204) -> (continuation of Deferred@548534655 after retry RPC after error@711269786) -> (continuation of Deferred@960581014 after retry RPC after error@822129155) -> (continuation of Deferred@517332829 after retry RPC after error@484350160) -> (continuation of Deferred@801135392 after retry RPC after error@467956849)))
20:28:10.892 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=221), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [43ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [161ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@833531004(state=PENDING, result=null, callback=(continuation of Deferred@37602281 after retry RPC after error@1056147148) -> (continuation of Deferred@677724785 after retry RPC after error@1827460685) -> (continuation of Deferred@10642098 after retry RPC after error@574968204) -> (continuation of Deferred@548534655 after retry RPC after error@711269786) -> (continuation of Deferred@960581014 after retry RPC after error@822129155) -> (continuation of Deferred@517332829 after retry RPC after error@484350160) -> (continuation of Deferred@801135392 after retry RPC after error@467956849), errback=(continuation of Deferred@37602281 after retry RPC after error@1056147148) -> (continuation of Deferred@677724785 after retry RPC after error@1827460685) -> (continuation of Deferred@10642098 after retry RPC after error@574968204) -> (continuation of Deferred@548534655 after retry RPC after error@711269786) -> (continuation of Deferred@960581014 after retry RPC after error@822129155) -> (continuation of Deferred@517332829 after retry RPC after error@484350160) -> (continuation of Deferred@801135392 after retry RPC after error@467956849)))
20:28:10.911 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:10.912 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:10.912 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:10.912 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:10.912 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:10.912 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:10.912 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:10.912 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:10.914 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:10.930 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:10.930 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:10.931 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:10.931 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:10.931 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:10.931 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:10.931 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:10.931 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:10.932 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:10.951 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:10.951 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:10.952 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:10.952 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:10.952 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:10.952 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:10.952 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:10.952 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:10.952 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:10.971 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:10.971 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:10.972 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:10.972 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:10.972 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:10.972 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:10.972 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:10.972 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:10.972 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:10.991 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:10.991 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:10.992 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:10.992 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:10.992 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:10.992 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:10.992 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:10.992 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:10.993 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:11.031 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:11.032 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:11.032 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:11.032 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:11.032 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:11.032 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:11.032 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:11.032 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:11.033 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:11.091 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:11.091 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:11.091 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:11.091 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:11.091 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:11.092 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:11.092 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:11.092 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:11.093 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:11.094 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [20ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [179ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [180ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@1056270340(state=PENDING, result=null, callback=(continuation of Deferred@702232304 after retry RPC after error@2080231901) -> (continuation of Deferred@1538702913 after retry RPC after error@1310087291) -> (continuation of Deferred@1049271295 after retry RPC after error@728553162) -> (continuation of Deferred@1162228048 after retry RPC after error@971996163) -> (continuation of Deferred@1406164386 after retry RPC after error@516365204) -> (continuation of Deferred@1795122605 after retry RPC after error@674370453), errback=(continuation of Deferred@702232304 after retry RPC after error@2080231901) -> (continuation of Deferred@1538702913 after retry RPC after error@1310087291) -> (continuation of Deferred@1049271295 after retry RPC after error@728553162) -> (continuation of Deferred@1162228048 after retry RPC after error@971996163) -> (continuation of Deferred@1406164386 after retry RPC after error@516365204) -> (continuation of Deferred@1795122605 after retry RPC after error@674370453)))
20:28:11.094 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [20ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [179ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [180ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@1056270340(state=PENDING, result=null, callback=(continuation of Deferred@702232304 after retry RPC after error@2080231901) -> (continuation of Deferred@1538702913 after retry RPC after error@1310087291) -> (continuation of Deferred@1049271295 after retry RPC after error@728553162) -> (continuation of Deferred@1162228048 after retry RPC after error@971996163) -> (continuation of Deferred@1406164386 after retry RPC after error@516365204) -> (continuation of Deferred@1795122605 after retry RPC after error@674370453), errback=(continuation of Deferred@702232304 after retry RPC after error@2080231901) -> (continuation of Deferred@1538702913 after retry RPC after error@1310087291) -> (continuation of Deferred@1049271295 after retry RPC after error@728553162) -> (continuation of Deferred@1162228048 after retry RPC after error@971996163) -> (continuation of Deferred@1406164386 after retry RPC after error@516365204) -> (continuation of Deferred@1795122605 after retry RPC after error@674370453)))
20:28:11.111 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:11.111 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:11.111 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:11.111 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:11.111 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:11.112 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:11.112 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:11.112 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:11.113 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:11.131 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:11.131 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:11.131 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:11.131 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:11.131 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:11.131 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:11.131 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:11.131 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:11.132 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:11.150 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:11.151 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:11.151 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:11.152 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:11.152 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:11.152 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:11.152 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:11.152 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:11.153 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:11.171 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:11.171 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:11.171 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:11.171 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:11.171 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:11.171 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:11.172 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:11.172 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:11.172 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:11.191 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:11.191 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:11.191 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:11.191 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:11.191 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:11.191 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:11.192 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:11.192 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:11.192 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:11.231 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:11.231 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:11.232 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:11.232 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:11.232 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:11.232 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:11.232 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:11.232 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:11.232 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:11.291 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:11.291 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:11.292 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:11.292 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:11.292 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:11.292 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:11.292 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:11.292 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:11.293 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:11.293 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [3ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [120ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [181ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [183ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@1319116084(state=PENDING, result=null, callback=(continuation of Deferred@1991639502 after retry RPC after error@2085745890) -> (continuation of Deferred@1027136025 after retry RPC after error@1933519159) -> (continuation of Deferred@901902063 after retry RPC after error@1187314257) -> (continuation of Deferred@1037506690 after retry RPC after error@925628574) -> (continuation of Deferred@229101058 after retry RPC after error@1817230229) -> (continuation of Deferred@1774628245 after retry RPC after error@29447995), errback=(continuation of Deferred@1991639502 after retry RPC after error@2085745890) -> (continuation of Deferred@1027136025 after retry RPC after error@1933519159) -> (continuation of Deferred@901902063 after retry RPC after error@1187314257) -> (continuation of Deferred@1037506690 after retry RPC after error@925628574) -> (continuation of Deferred@229101058 after retry RPC after error@1817230229) -> (continuation of Deferred@1774628245 after retry RPC after error@29447995)))
20:28:11.293 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [3ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [120ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [181ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [183ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@1319116084(state=PENDING, result=null, callback=(continuation of Deferred@1991639502 after retry RPC after error@2085745890) -> (continuation of Deferred@1027136025 after retry RPC after error@1933519159) -> (continuation of Deferred@901902063 after retry RPC after error@1187314257) -> (continuation of Deferred@1037506690 after retry RPC after error@925628574) -> (continuation of Deferred@229101058 after retry RPC after error@1817230229) -> (continuation of Deferred@1774628245 after retry RPC after error@29447995), errback=(continuation of Deferred@1991639502 after retry RPC after error@2085745890) -> (continuation of Deferred@1027136025 after retry RPC after error@1933519159) -> (continuation of Deferred@901902063 after retry RPC after error@1187314257) -> (continuation of Deferred@1037506690 after retry RPC after error@925628574) -> (continuation of Deferred@229101058 after retry RPC after error@1817230229) -> (continuation of Deferred@1774628245 after retry RPC after error@29447995)))
20:28:11.311 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:11.311 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:11.312 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:11.312 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:11.312 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:11.312 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:11.312 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:11.312 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:11.313 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:11.331 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:11.331 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:11.331 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:11.331 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:11.331 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:11.331 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:11.332 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:11.332 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:11.332 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:11.351 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:11.351 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:11.351 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:11.351 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:11.352 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:11.352 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:11.352 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:11.352 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:11.352 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:11.371 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:11.371 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:11.371 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:11.371 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:11.371 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:11.371 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:11.372 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:11.372 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:11.372 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:11.390 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:11.391 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:11.391 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:11.391 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:11.391 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:11.391 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:11.391 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:11.391 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:11.392 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:11.410 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:11.411 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:11.411 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:11.411 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:11.411 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:11.411 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:11.411 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:11.411 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:11.412 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:11.450 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:11.451 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:11.451 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:11.451 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:11.451 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:11.451 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:11.451 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:11.451 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:11.452 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:11.452 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=141), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [19ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [20ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [60ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [80ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [99ms] refreshing cache from master, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [100ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [100ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [101ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [139ms] refreshing cache from master, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [140ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [140ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [141ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@336490566(state=PENDING, result=null, callback=(continuation of Deferred@1227601269 after retry RPC after error@2100743918) -> (continuation of Deferred@1576406339 after retry RPC after error@975035131) -> (continuation of Deferred@780751056 after retry RPC after error@1694743121) -> (continuation of Deferred@150093735 after retry RPC after error@1550644800) -> (continuation of Deferred@948380365 after retry RPC after error@1076811454) -> (continuation of Deferred@1505281563 after retry RPC after error@1650320794), errback=(continuation of Deferred@1227601269 after retry RPC after error@2100743918) -> (continuation of Deferred@1576406339 after retry RPC after error@975035131) -> (continuation of Deferred@780751056 after retry RPC after error@1694743121) -> (continuation of Deferred@150093735 after retry RPC after error@1550644800) -> (continuation of Deferred@948380365 after retry RPC after error@1076811454) -> (continuation of Deferred@1505281563 after retry RPC after error@1650320794)))
20:28:11.452 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=141), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [19ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [20ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [60ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [80ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [99ms] refreshing cache from master, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [100ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [100ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [101ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [139ms] refreshing cache from master, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [140ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [140ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [141ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@336490566(state=PENDING, result=null, callback=(continuation of Deferred@1227601269 after retry RPC after error@2100743918) -> (continuation of Deferred@1576406339 after retry RPC after error@975035131) -> (continuation of Deferred@780751056 after retry RPC after error@1694743121) -> (continuation of Deferred@150093735 after retry RPC after error@1550644800) -> (continuation of Deferred@948380365 after retry RPC after error@1076811454) -> (continuation of Deferred@1505281563 after retry RPC after error@1650320794), errback=(continuation of Deferred@1227601269 after retry RPC after error@2100743918) -> (continuation of Deferred@1576406339 after retry RPC after error@975035131) -> (continuation of Deferred@780751056 after retry RPC after error@1694743121) -> (continuation of Deferred@150093735 after retry RPC after error@1550644800) -> (continuation of Deferred@948380365 after retry RPC after error@1076811454) -> (continuation of Deferred@1505281563 after retry RPC after error@1650320794)))
20:28:11.470 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:11.471 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:11.471 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:11.471 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:11.471 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:11.471 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:11.471 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:11.471 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:11.472 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:11.490 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:11.491 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:11.491 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:11.491 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:11.491 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:11.491 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:11.491 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:11.492 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:11.492 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:11.511 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:11.511 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:11.511 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:11.511 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:11.512 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:11.512 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:11.512 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:11.512 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:11.512 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:11.531 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:11.531 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:11.531 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:11.531 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:11.531 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:11.532 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:11.532 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:11.532 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:11.532 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:11.551 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:11.551 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:11.551 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:11.551 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:11.551 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:11.551 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:11.551 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:11.551 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:11.552 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:11.571 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:11.571 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:11.571 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:11.571 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:11.571 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:11.571 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:11.571 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:11.571 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:11.572 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:11.590 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:11.591 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:11.591 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:11.591 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:11.591 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:11.591 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:11.591 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:11.591 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:11.592 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:11.671 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:11.671 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:11.671 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:11.672 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:11.672 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:11.672 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:11.672 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:11.672 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:11.672 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:11.673 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [200ms] refreshing cache from master, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@1306396618(state=PENDING, result=null, callback=(continuation of Deferred@477251709 after retry RPC after error@1846487938) -> (continuation of Deferred@1930297478 after retry RPC after error@1322494330) -> (continuation of Deferred@1089216893 after retry RPC after error@1462260185) -> (continuation of Deferred@1419862410 after retry RPC after error@124426781) -> (continuation of Deferred@640459844 after retry RPC after error@866099137) -> (continuation of Deferred@1827472649 after retry RPC after error@1065668859) -> (continuation of Deferred@2093571531 after retry RPC after error@862430212), errback=(continuation of Deferred@477251709 after retry RPC after error@1846487938) -> (continuation of Deferred@1930297478 after retry RPC after error@1322494330) -> (continuation of Deferred@1089216893 after retry RPC after error@1462260185) -> (continuation of Deferred@1419862410 after retry RPC after error@124426781) -> (continuation of Deferred@640459844 after retry RPC after error@866099137) -> (continuation of Deferred@1827472649 after retry RPC after error@1065668859) -> (continuation of Deferred@2093571531 after retry RPC after error@862430212)))
20:28:11.673 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [200ms] refreshing cache from master, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@1306396618(state=PENDING, result=null, callback=(continuation of Deferred@477251709 after retry RPC after error@1846487938) -> (continuation of Deferred@1930297478 after retry RPC after error@1322494330) -> (continuation of Deferred@1089216893 after retry RPC after error@1462260185) -> (continuation of Deferred@1419862410 after retry RPC after error@124426781) -> (continuation of Deferred@640459844 after retry RPC after error@866099137) -> (continuation of Deferred@1827472649 after retry RPC after error@1065668859) -> (continuation of Deferred@2093571531 after retry RPC after error@862430212), errback=(continuation of Deferred@477251709 after retry RPC after error@1846487938) -> (continuation of Deferred@1930297478 after retry RPC after error@1322494330) -> (continuation of Deferred@1089216893 after retry RPC after error@1462260185) -> (continuation of Deferred@1419862410 after retry RPC after error@124426781) -> (continuation of Deferred@640459844 after retry RPC after error@866099137) -> (continuation of Deferred@1827472649 after retry RPC after error@1065668859) -> (continuation of Deferred@2093571531 after retry RPC after error@862430212)))
20:28:11.691 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:11.691 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:11.691 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:11.691 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:11.691 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:11.692 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:11.692 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:11.692 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:11.692 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:11.711 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:11.711 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:11.711 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:11.711 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:11.711 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:11.711 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:11.711 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:11.711 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:11.712 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:11.730 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:11.731 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:11.731 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:11.731 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:11.731 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:11.731 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:11.731 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:11.731 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:11.732 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:11.750 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:11.751 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:11.751 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:11.751 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:11.751 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:11.751 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:11.751 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:11.751 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:11.752 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:11.770 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:11.770 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:11.771 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:11.771 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:11.771 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:11.771 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:11.771 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:11.771 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:11.771 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:11.791 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:11.791 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:11.792 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:11.792 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:11.792 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:11.792 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:11.792 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:11.792 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:11.793 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:11.850 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:11.851 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:11.851 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:11.851 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:11.851 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:11.851 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:11.851 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:11.851 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:11.852 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:11.852 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=161), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [103ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@269840157(state=PENDING, result=null, callback=(continuation of Deferred@1600067531 after retry RPC after error@797126802) -> (continuation of Deferred@525483197 after retry RPC after error@604292557) -> (continuation of Deferred@54689737 after retry RPC after error@896784531) -> (continuation of Deferred@1003142484 after retry RPC after error@962739533) -> (continuation of Deferred@701407647 after retry RPC after error@47334261) -> (continuation of Deferred@1822195302 after retry RPC after error@1153216237), errback=(continuation of Deferred@1600067531 after retry RPC after error@797126802) -> (continuation of Deferred@525483197 after retry RPC after error@604292557) -> (continuation of Deferred@54689737 after retry RPC after error@896784531) -> (continuation of Deferred@1003142484 after retry RPC after error@962739533) -> (continuation of Deferred@701407647 after retry RPC after error@47334261) -> (continuation of Deferred@1822195302 after retry RPC after error@1153216237)))
20:28:11.852 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=161), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [103ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@269840157(state=PENDING, result=null, callback=(continuation of Deferred@1600067531 after retry RPC after error@797126802) -> (continuation of Deferred@525483197 after retry RPC after error@604292557) -> (continuation of Deferred@54689737 after retry RPC after error@896784531) -> (continuation of Deferred@1003142484 after retry RPC after error@962739533) -> (continuation of Deferred@701407647 after retry RPC after error@47334261) -> (continuation of Deferred@1822195302 after retry RPC after error@1153216237), errback=(continuation of Deferred@1600067531 after retry RPC after error@797126802) -> (continuation of Deferred@525483197 after retry RPC after error@604292557) -> (continuation of Deferred@54689737 after retry RPC after error@896784531) -> (continuation of Deferred@1003142484 after retry RPC after error@962739533) -> (continuation of Deferred@701407647 after retry RPC after error@47334261) -> (continuation of Deferred@1822195302 after retry RPC after error@1153216237)))
20:28:11.870 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:11.871 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:11.871 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:11.871 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:11.871 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:11.871 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:11.871 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:11.872 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:11.872 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:11.890 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:11.893 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:11.893 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:11.893 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:11.893 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:11.893 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:11.894 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:11.894 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:11.894 [WARN - kudu-nio-7] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:11.910 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:11.911 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:11.911 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:11.911 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:11.911 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:11.911 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:11.911 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:11.911 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:11.912 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:11.930 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:11.931 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:11.931 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:11.931 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:11.931 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:11.931 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:11.931 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:11.931 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:11.932 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:11.950 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:11.951 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:11.951 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:11.951 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:11.951 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:11.951 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:11.951 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:11.951 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:11.952 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:11.970 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:11.970 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:11.971 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:11.971 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:11.971 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:11.971 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:11.971 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:11.971 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:11.972 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:11.991 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:11.991 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:11.992 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:11.992 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:11.992 [INFO - pool-40-thread-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:11.993 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:11.993 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:11.993 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:11.993 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:12.051 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:12.051 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:12.052 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:12.052 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:12.052 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:12.052 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:12.052 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:12.052 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:12.052 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:12.091 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:12.091 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:12.091 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:12.091 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:12.091 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:12.091 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:12.091 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:12.092 [INFO - pool-40-thread-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:12.093 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:12.093 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [23ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [23ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [23ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [24ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [24ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [123ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [123ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [123ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [181ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [223ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [223ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@792147814(state=PENDING, result=null, callback=(continuation of Deferred@2147292170 after retry RPC after error@567858368) -> (continuation of Deferred@235456075 after retry RPC after error@403852523) -> (continuation of Deferred@265832103 after retry RPC after error@1757539656) -> (continuation of Deferred@548619397 after retry RPC after error@718020972) -> (continuation of Deferred@1743566828 after retry RPC after error@2141684) -> (continuation of Deferred@973948441 after retry RPC after error@1904330613) -> (continuation of Deferred@1934642558 after retry RPC after error@1989808621) -> (continuation of Deferred@653765801 after retry RPC after error@1496481133), errback=(continuation of Deferred@2147292170 after retry RPC after error@567858368) -> (continuation of Deferred@235456075 after retry RPC after error@403852523) -> (continuation of Deferred@265832103 after retry RPC after error@1757539656) -> (continuation of Deferred@548619397 after retry RPC after error@718020972) -> (continuation of Deferred@1743566828 after retry RPC after error@2141684) -> (continuation of Deferred@973948441 after retry RPC after error@1904330613) -> (continuation of Deferred@1934642558 after retry RPC after error@1989808621) -> (continuation of Deferred@653765801 after retry RPC after error@1496481133)))
20:28:12.094 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [23ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [23ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [23ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [24ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [24ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [123ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [123ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [123ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [181ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [223ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [223ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@792147814(state=PENDING, result=null, callback=(continuation of Deferred@2147292170 after retry RPC after error@567858368) -> (continuation of Deferred@235456075 after retry RPC after error@403852523) -> (continuation of Deferred@265832103 after retry RPC after error@1757539656) -> (continuation of Deferred@548619397 after retry RPC after error@718020972) -> (continuation of Deferred@1743566828 after retry RPC after error@2141684) -> (continuation of Deferred@973948441 after retry RPC after error@1904330613) -> (continuation of Deferred@1934642558 after retry RPC after error@1989808621) -> (continuation of Deferred@653765801 after retry RPC after error@1496481133), errback=(continuation of Deferred@2147292170 after retry RPC after error@567858368) -> (continuation of Deferred@235456075 after retry RPC after error@403852523) -> (continuation of Deferred@265832103 after retry RPC after error@1757539656) -> (continuation of Deferred@548619397 after retry RPC after error@718020972) -> (continuation of Deferred@1743566828 after retry RPC after error@2141684) -> (continuation of Deferred@973948441 after retry RPC after error@1904330613) -> (continuation of Deferred@1934642558 after retry RPC after error@1989808621) -> (continuation of Deferred@653765801 after retry RPC after error@1496481133)))
20:28:12.111 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:12.111 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:12.112 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:12.112 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:12.112 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:12.113 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:12.113 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:12.113 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:12.114 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:12.130 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:12.131 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:12.131 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:12.131 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:12.131 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:12.131 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:12.131 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:12.131 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:12.132 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:12.150 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:12.151 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:12.151 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:12.151 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:12.151 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:12.151 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:12.151 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:12.151 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:12.152 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:12.170 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:12.170 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:12.171 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:12.171 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:12.171 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:12.171 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:12.171 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:12.171 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:12.171 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:12.171 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:12.172 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:12.172 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:12.190 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:12.190 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:12.191 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:12.191 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:12.191 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:12.191 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:12.191 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:12.191 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:12.192 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:12.211 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:12.211 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:12.212 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:12.213 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:12.213 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:12.213 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:12.213 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:12.213 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:12.213 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:12.270 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:12.271 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:12.271 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:12.271 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:12.271 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:12.271 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:12.271 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:12.271 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:12.272 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:12.272 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=161), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [3ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [20ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [40ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [60ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [80ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [159ms] refreshing cache from master, [159ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [159ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [159ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [160ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [160ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [161ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@818046853(state=PENDING, result=null, callback=(continuation of Deferred@1645437839 after retry RPC after error@1613205388) -> (continuation of Deferred@908249287 after retry RPC after error@369061833) -> (continuation of Deferred@1820944866 after retry RPC after error@806273040) -> (continuation of Deferred@281694758 after retry RPC after error@2116413137) -> (continuation of Deferred@1497646734 after retry RPC after error@520881142) -> (continuation of Deferred@1570060627 after retry RPC after error@207052638), errback=(continuation of Deferred@1645437839 after retry RPC after error@1613205388) -> (continuation of Deferred@908249287 after retry RPC after error@369061833) -> (continuation of Deferred@1820944866 after retry RPC after error@806273040) -> (continuation of Deferred@281694758 after retry RPC after error@2116413137) -> (continuation of Deferred@1497646734 after retry RPC after error@520881142) -> (continuation of Deferred@1570060627 after retry RPC after error@207052638)))
20:28:12.272 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=161), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [3ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [20ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [40ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [60ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [80ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [159ms] refreshing cache from master, [159ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [159ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [159ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [160ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [160ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [161ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@818046853(state=PENDING, result=null, callback=(continuation of Deferred@1645437839 after retry RPC after error@1613205388) -> (continuation of Deferred@908249287 after retry RPC after error@369061833) -> (continuation of Deferred@1820944866 after retry RPC after error@806273040) -> (continuation of Deferred@281694758 after retry RPC after error@2116413137) -> (continuation of Deferred@1497646734 after retry RPC after error@520881142) -> (continuation of Deferred@1570060627 after retry RPC after error@207052638), errback=(continuation of Deferred@1645437839 after retry RPC after error@1613205388) -> (continuation of Deferred@908249287 after retry RPC after error@369061833) -> (continuation of Deferred@1820944866 after retry RPC after error@806273040) -> (continuation of Deferred@281694758 after retry RPC after error@2116413137) -> (continuation of Deferred@1497646734 after retry RPC after error@520881142) -> (continuation of Deferred@1570060627 after retry RPC after error@207052638)))
20:28:12.290 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:12.290 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:12.291 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:12.291 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:12.291 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:12.291 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:12.291 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:12.291 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:12.292 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:12.311 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:12.311 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:12.311 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:12.311 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:12.312 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:12.312 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:12.312 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:12.312 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:12.312 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:12.331 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:12.331 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:12.331 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:12.331 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:12.332 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:12.332 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:12.332 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:12.332 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:12.332 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:12.350 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:12.351 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:12.351 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:12.351 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:12.351 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:12.351 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:12.351 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:12.351 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:12.352 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:12.370 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:12.371 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:12.371 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:12.371 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:12.371 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:12.371 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:12.371 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:12.371 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:12.372 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:12.390 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:12.390 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:12.391 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:12.391 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:12.391 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:12.391 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:12.391 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:12.391 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:12.392 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:12.450 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:12.451 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:12.451 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:12.451 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:12.451 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:12.451 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:12.451 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:12.451 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:12.452 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:12.470 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:12.471 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:12.471 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:12.471 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:12.471 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:12.471 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:12.471 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:12.472 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:12.472 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:12.472 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [40ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@1560853008(state=PENDING, result=null, callback=(continuation of Deferred@1342411221 after retry RPC after error@1995907098) -> (continuation of Deferred@1042114788 after retry RPC after error@652057869) -> (continuation of Deferred@1788624186 after retry RPC after error@631519107) -> (continuation of Deferred@1890267547 after retry RPC after error@987033462) -> (continuation of Deferred@824860923 after retry RPC after error@901140850) -> (continuation of Deferred@473467318 after retry RPC after error@25852440) -> (continuation of Deferred@1673916266 after retry RPC after error@1694441377), errback=(continuation of Deferred@1342411221 after retry RPC after error@1995907098) -> (continuation of Deferred@1042114788 after retry RPC after error@652057869) -> (continuation of Deferred@1788624186 after retry RPC after error@631519107) -> (continuation of Deferred@1890267547 after retry RPC after error@987033462) -> (continuation of Deferred@824860923 after retry RPC after error@901140850) -> (continuation of Deferred@473467318 after retry RPC after error@25852440) -> (continuation of Deferred@1673916266 after retry RPC after error@1694441377)))
20:28:12.473 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [40ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@1560853008(state=PENDING, result=null, callback=(continuation of Deferred@1342411221 after retry RPC after error@1995907098) -> (continuation of Deferred@1042114788 after retry RPC after error@652057869) -> (continuation of Deferred@1788624186 after retry RPC after error@631519107) -> (continuation of Deferred@1890267547 after retry RPC after error@987033462) -> (continuation of Deferred@824860923 after retry RPC after error@901140850) -> (continuation of Deferred@473467318 after retry RPC after error@25852440) -> (continuation of Deferred@1673916266 after retry RPC after error@1694441377), errback=(continuation of Deferred@1342411221 after retry RPC after error@1995907098) -> (continuation of Deferred@1042114788 after retry RPC after error@652057869) -> (continuation of Deferred@1788624186 after retry RPC after error@631519107) -> (continuation of Deferred@1890267547 after retry RPC after error@987033462) -> (continuation of Deferred@824860923 after retry RPC after error@901140850) -> (continuation of Deferred@473467318 after retry RPC after error@25852440) -> (continuation of Deferred@1673916266 after retry RPC after error@1694441377)))
20:28:12.490 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:12.491 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:12.491 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:12.491 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:12.491 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:12.491 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:12.491 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:12.491 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:12.492 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:12.510 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:12.511 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:12.511 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:12.511 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:12.511 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:12.512 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:12.512 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:12.512 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:12.512 [WARN - kudu-nio-7] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:12.531 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:12.531 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:12.532 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:12.532 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:12.532 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:12.532 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:12.532 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:12.532 [INFO - pool-40-thread-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:12.532 [WARN - pool-40-thread-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:12.551 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:12.551 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:12.551 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:12.552 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:12.552 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:12.552 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:12.552 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:12.552 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:12.552 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:12.570 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:12.570 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:12.571 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:12.571 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:12.571 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:12.571 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:12.571 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:12.571 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:12.571 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:12.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:28:12.582837 3893 master_proxy_rpc.cc:177] Failed to determine new leader Master: Timed out: ConnectToClusterRpc(addrs: 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127, num_attempts: 97) passed its deadline: Not found: no leader found: ConnectToClusterRpc(addrs: 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127, num_attempts: 1)
20:28:12.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:28:12.584306 3893 meta_cache.cc:788] Timed out: LookupRpcById { tablet: 'f9ccfdb32e164e61bde68775baab7f9d', attempt: 2 } failed: LookupRpcById timed out after deadline expired: LookupRpcById { tablet: 'f9ccfdb32e164e61bde68775baab7f9d', attempt: 2 } passed its deadline: Timed out: ConnectToClusterRpc(addrs: 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127, num_attempts: 97) passed its deadline: Not found: no leader found: ConnectToClusterRpc(addrs: 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127, num_attempts: 1)
20:28:12.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:28:12.586757 3893 master_proxy_rpc.cc:203] Re-attempting LookupRpcById request to leader Master (127.3.46.62:46071)
20:28:12.590 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:12.591 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:12.591 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:12.591 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:12.591 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:12.591 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:12.591 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:12.591 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:12.592 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:12.651 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:12.651 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:12.651 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:12.651 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:12.651 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:12.651 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:12.652 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:12.652 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:12.652 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:12.652 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=161), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@280480452(state=PENDING, result=null, callback=(continuation of Deferred@731035179 after retry RPC after error@1283011410) -> (continuation of Deferred@1198465580 after retry RPC after error@1763713519) -> (continuation of Deferred@1472065806 after retry RPC after error@1381743368) -> (continuation of Deferred@1813744941 after retry RPC after error@81589784) -> (continuation of Deferred@1011962380 after retry RPC after error@861744501) -> (continuation of Deferred@60796118 after retry RPC after error@1620113035), errback=(continuation of Deferred@731035179 after retry RPC after error@1283011410) -> (continuation of Deferred@1198465580 after retry RPC after error@1763713519) -> (continuation of Deferred@1472065806 after retry RPC after error@1381743368) -> (continuation of Deferred@1813744941 after retry RPC after error@81589784) -> (continuation of Deferred@1011962380 after retry RPC after error@861744501) -> (continuation of Deferred@60796118 after retry RPC after error@1620113035)))
20:28:12.653 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=161), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@280480452(state=PENDING, result=null, callback=(continuation of Deferred@731035179 after retry RPC after error@1283011410) -> (continuation of Deferred@1198465580 after retry RPC after error@1763713519) -> (continuation of Deferred@1472065806 after retry RPC after error@1381743368) -> (continuation of Deferred@1813744941 after retry RPC after error@81589784) -> (continuation of Deferred@1011962380 after retry RPC after error@861744501) -> (continuation of Deferred@60796118 after retry RPC after error@1620113035), errback=(continuation of Deferred@731035179 after retry RPC after error@1283011410) -> (continuation of Deferred@1198465580 after retry RPC after error@1763713519) -> (continuation of Deferred@1472065806 after retry RPC after error@1381743368) -> (continuation of Deferred@1813744941 after retry RPC after error@81589784) -> (continuation of Deferred@1011962380 after retry RPC after error@861744501) -> (continuation of Deferred@60796118 after retry RPC after error@1620113035)))
20:28:12.671 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:12.671 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:12.672 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:12.672 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:12.672 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:12.672 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:12.672 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:12.672 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:12.673 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:12.691 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:12.691 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:12.692 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:12.692 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:12.692 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:12.692 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:12.692 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:12.692 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:12.692 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:12.710 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:12.711 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:12.712 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:12.712 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:12.712 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:12.712 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:12.712 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:12.712 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:12.712 [WARN - kudu-nio-7] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:12.730 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:12.731 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:12.731 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:12.731 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:12.731 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:12.731 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:12.732 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:12.732 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:12.732 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:12.751 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:12.751 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:12.752 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:12.752 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:12.752 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:12.752 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:12.752 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:12.752 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:12.753 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:12.771 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:12.771 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:12.772 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:12.772 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:12.772 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:12.772 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:12.772 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:12.772 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:12.773 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:12.790 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:12.791 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:12.791 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:12.791 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:12.791 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:12.791 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:12.791 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:12.791 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:12.792 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:12.911 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:12.911 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:12.912 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:12.912 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:12.912 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:12.912 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:12.912 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:12.912 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:12.913 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:12.913 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=241), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [60ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [119ms] refreshing cache from master, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [120ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [120ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [121ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@2064341403(state=PENDING, result=null, callback=(continuation of Deferred@710222217 after retry RPC after error@980404935) -> (continuation of Deferred@1364125662 after retry RPC after error@1890102253) -> (continuation of Deferred@1945277720 after retry RPC after error@923750483) -> (continuation of Deferred@291053947 after retry RPC after error@614139774) -> (continuation of Deferred@1126571168 after retry RPC after error@241586527) -> (continuation of Deferred@1449978641 after retry RPC after error@716219989) -> (continuation of Deferred@1493200063 after retry RPC after error@1699754601), errback=(continuation of Deferred@710222217 after retry RPC after error@980404935) -> (continuation of Deferred@1364125662 after retry RPC after error@1890102253) -> (continuation of Deferred@1945277720 after retry RPC after error@923750483) -> (continuation of Deferred@291053947 after retry RPC after error@614139774) -> (continuation of Deferred@1126571168 after retry RPC after error@241586527) -> (continuation of Deferred@1449978641 after retry RPC after error@716219989) -> (continuation of Deferred@1493200063 after retry RPC after error@1699754601)))
20:28:12.913 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=241), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [60ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [119ms] refreshing cache from master, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [120ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [120ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [121ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@2064341403(state=PENDING, result=null, callback=(continuation of Deferred@710222217 after retry RPC after error@980404935) -> (continuation of Deferred@1364125662 after retry RPC after error@1890102253) -> (continuation of Deferred@1945277720 after retry RPC after error@923750483) -> (continuation of Deferred@291053947 after retry RPC after error@614139774) -> (continuation of Deferred@1126571168 after retry RPC after error@241586527) -> (continuation of Deferred@1449978641 after retry RPC after error@716219989) -> (continuation of Deferred@1493200063 after retry RPC after error@1699754601), errback=(continuation of Deferred@710222217 after retry RPC after error@980404935) -> (continuation of Deferred@1364125662 after retry RPC after error@1890102253) -> (continuation of Deferred@1945277720 after retry RPC after error@923750483) -> (continuation of Deferred@291053947 after retry RPC after error@614139774) -> (continuation of Deferred@1126571168 after retry RPC after error@241586527) -> (continuation of Deferred@1449978641 after retry RPC after error@716219989) -> (continuation of Deferred@1493200063 after retry RPC after error@1699754601)))
20:28:12.931 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:12.931 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:12.931 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:12.931 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:12.931 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:12.931 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:12.931 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:12.931 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:12.932 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:12.950 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:12.951 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:12.951 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:12.951 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:12.951 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:12.951 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:12.951 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:12.951 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:12.952 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:12.970 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:12.971 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:12.971 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:12.971 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:12.971 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:12.971 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:12.971 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:12.971 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:12.972 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:12.990 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:12.990 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:12.991 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:12.991 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:12.991 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:12.991 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:12.991 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:12.991 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:12.992 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:13.011 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:13.011 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:13.012 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:13.012 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:13.012 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:13.012 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:13.012 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:13.012 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:13.012 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:13.031 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:13.031 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:13.031 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:13.031 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:13.031 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:13.031 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:13.031 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:13.031 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:13.032 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:13.091 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:13.091 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:13.091 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:13.091 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:13.091 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:13.091 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:13.091 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:13.092 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:13.092 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:13.151 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:13.151 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:13.152 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:13.152 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:13.152 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:13.152 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:13.152 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:13.152 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:13.153 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:13.153 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [100ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [160ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [221ms] refreshing cache from master, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [222ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [222ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [222ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [223ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@535808494(state=PENDING, result=null, callback=(continuation of Deferred@1605307992 after retry RPC after error@1531131915) -> (continuation of Deferred@478767129 after retry RPC after error@2110284511) -> (continuation of Deferred@1513258736 after retry RPC after error@410228445) -> (continuation of Deferred@1332604010 after retry RPC after error@1905107394) -> (continuation of Deferred@948016381 after retry RPC after error@1010841678) -> (continuation of Deferred@477011216 after retry RPC after error@15842542) -> (continuation of Deferred@878966385 after retry RPC after error@157039355), errback=(continuation of Deferred@1605307992 after retry RPC after error@1531131915) -> (continuation of Deferred@478767129 after retry RPC after error@2110284511) -> (continuation of Deferred@1513258736 after retry RPC after error@410228445) -> (continuation of Deferred@1332604010 after retry RPC after error@1905107394) -> (continuation of Deferred@948016381 after retry RPC after error@1010841678) -> (continuation of Deferred@477011216 after retry RPC after error@15842542) -> (continuation of Deferred@878966385 after retry RPC after error@157039355)))
20:28:13.153 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [100ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [160ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [221ms] refreshing cache from master, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [222ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [222ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [222ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [223ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@535808494(state=PENDING, result=null, callback=(continuation of Deferred@1605307992 after retry RPC after error@1531131915) -> (continuation of Deferred@478767129 after retry RPC after error@2110284511) -> (continuation of Deferred@1513258736 after retry RPC after error@410228445) -> (continuation of Deferred@1332604010 after retry RPC after error@1905107394) -> (continuation of Deferred@948016381 after retry RPC after error@1010841678) -> (continuation of Deferred@477011216 after retry RPC after error@15842542) -> (continuation of Deferred@878966385 after retry RPC after error@157039355), errback=(continuation of Deferred@1605307992 after retry RPC after error@1531131915) -> (continuation of Deferred@478767129 after retry RPC after error@2110284511) -> (continuation of Deferred@1513258736 after retry RPC after error@410228445) -> (continuation of Deferred@1332604010 after retry RPC after error@1905107394) -> (continuation of Deferred@948016381 after retry RPC after error@1010841678) -> (continuation of Deferred@477011216 after retry RPC after error@15842542) -> (continuation of Deferred@878966385 after retry RPC after error@157039355)))
20:28:13.171 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:13.171 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:13.172 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:13.172 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:13.172 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:13.172 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:13.172 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:13.172 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:13.172 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:13.191 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:13.191 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:13.192 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:13.192 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:13.192 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:13.192 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:13.192 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:13.192 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:13.192 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:13.211 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:13.211 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:13.212 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:13.212 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:13.212 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:13.212 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:13.212 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:13.212 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:13.213 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:13.231 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:13.231 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:13.232 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:13.232 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:13.232 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:13.232 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:13.232 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:13.232 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:13.233 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:13.251 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:13.251 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:13.251 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:13.251 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:13.251 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:13.251 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:13.252 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:13.252 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:13.252 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:13.271 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:13.271 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:13.271 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:13.271 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:13.271 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:13.271 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:13.271 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:13.271 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:13.272 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:13.290 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:13.291 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:13.291 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:13.291 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:13.291 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:13.291 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:13.291 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:13.291 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:13.292 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:13.330 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:13.331 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:13.331 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:13.331 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:13.331 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:13.331 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:13.331 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:13.331 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:13.332 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:13.332 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=161), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [1ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [79ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [80ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [99ms] refreshing cache from master, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [100ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [100ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [101ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [119ms] refreshing cache from master, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [120ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [120ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [121ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [159ms] refreshing cache from master, [159ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [159ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [160ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [160ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [161ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@1534152147(state=PENDING, result=null, callback=(continuation of Deferred@2085218250 after retry RPC after error@761290061) -> (continuation of Deferred@265709685 after retry RPC after error@1587065737) -> (continuation of Deferred@1834772519 after retry RPC after error@1123670772) -> (continuation of Deferred@2119206686 after retry RPC after error@1699706790) -> (continuation of Deferred@1809593976 after retry RPC after error@1736343437) -> (continuation of Deferred@502033419 after retry RPC after error@50383447) -> (continuation of Deferred@157631417 after retry RPC after error@494126863), errback=(continuation of Deferred@2085218250 after retry RPC after error@761290061) -> (continuation of Deferred@265709685 after retry RPC after error@1587065737) -> (continuation of Deferred@1834772519 after retry RPC after error@1123670772) -> (continuation of Deferred@2119206686 after retry RPC after error@1699706790) -> (continuation of Deferred@1809593976 after retry RPC after error@1736343437) -> (continuation of Deferred@502033419 after retry RPC after error@50383447) -> (continuation of Deferred@157631417 after retry RPC after error@494126863)))
20:28:13.333 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=161), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [1ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [79ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [80ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [99ms] refreshing cache from master, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [100ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [100ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [101ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [119ms] refreshing cache from master, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [120ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [120ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [121ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [159ms] refreshing cache from master, [159ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [159ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [160ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [160ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [161ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@1534152147(state=PENDING, result=null, callback=(continuation of Deferred@2085218250 after retry RPC after error@761290061) -> (continuation of Deferred@265709685 after retry RPC after error@1587065737) -> (continuation of Deferred@1834772519 after retry RPC after error@1123670772) -> (continuation of Deferred@2119206686 after retry RPC after error@1699706790) -> (continuation of Deferred@1809593976 after retry RPC after error@1736343437) -> (continuation of Deferred@502033419 after retry RPC after error@50383447) -> (continuation of Deferred@157631417 after retry RPC after error@494126863), errback=(continuation of Deferred@2085218250 after retry RPC after error@761290061) -> (continuation of Deferred@265709685 after retry RPC after error@1587065737) -> (continuation of Deferred@1834772519 after retry RPC after error@1123670772) -> (continuation of Deferred@2119206686 after retry RPC after error@1699706790) -> (continuation of Deferred@1809593976 after retry RPC after error@1736343437) -> (continuation of Deferred@502033419 after retry RPC after error@50383447) -> (continuation of Deferred@157631417 after retry RPC after error@494126863)))
20:28:13.350 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:13.351 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:13.351 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:13.351 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:13.351 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:13.351 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:13.351 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:13.351 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:13.352 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:13.370 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:13.370 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:13.371 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:13.371 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:13.371 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:13.371 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:13.371 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:13.371 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:13.372 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:13.391 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:13.391 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:13.392 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:13.392 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:13.392 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:13.392 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:13.392 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:13.392 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:13.392 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:13.411 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:13.411 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:13.412 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:13.412 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:13.412 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:13.412 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:13.412 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:13.412 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:13.413 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:13.431 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:13.431 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:13.431 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:13.431 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:13.431 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:13.431 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:13.432 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:13.432 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:13.432 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:13.450 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:13.451 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:13.451 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:13.451 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:13.451 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:13.451 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:13.451 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:13.451 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:13.452 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:13.510 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:13.511 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:13.511 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:13.511 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:13.511 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:13.511 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:13.511 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:13.511 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:13.512 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:13.512 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [63ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@1158528819(state=PENDING, result=null, callback=(continuation of Deferred@1187336994 after retry RPC after error@784872168) -> (continuation of Deferred@2060218226 after retry RPC after error@1385214783) -> (continuation of Deferred@1050719756 after retry RPC after error@1360038978) -> (continuation of Deferred@1220464365 after retry RPC after error@534807950) -> (continuation of Deferred@608911528 after retry RPC after error@1911339775) -> (continuation of Deferred@1335614345 after retry RPC after error@1414331137), errback=(continuation of Deferred@1187336994 after retry RPC after error@784872168) -> (continuation of Deferred@2060218226 after retry RPC after error@1385214783) -> (continuation of Deferred@1050719756 after retry RPC after error@1360038978) -> (continuation of Deferred@1220464365 after retry RPC after error@534807950) -> (continuation of Deferred@608911528 after retry RPC after error@1911339775) -> (continuation of Deferred@1335614345 after retry RPC after error@1414331137)))
20:28:13.513 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [63ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@1158528819(state=PENDING, result=null, callback=(continuation of Deferred@1187336994 after retry RPC after error@784872168) -> (continuation of Deferred@2060218226 after retry RPC after error@1385214783) -> (continuation of Deferred@1050719756 after retry RPC after error@1360038978) -> (continuation of Deferred@1220464365 after retry RPC after error@534807950) -> (continuation of Deferred@608911528 after retry RPC after error@1911339775) -> (continuation of Deferred@1335614345 after retry RPC after error@1414331137), errback=(continuation of Deferred@1187336994 after retry RPC after error@784872168) -> (continuation of Deferred@2060218226 after retry RPC after error@1385214783) -> (continuation of Deferred@1050719756 after retry RPC after error@1360038978) -> (continuation of Deferred@1220464365 after retry RPC after error@534807950) -> (continuation of Deferred@608911528 after retry RPC after error@1911339775) -> (continuation of Deferred@1335614345 after retry RPC after error@1414331137)))
20:28:13.530 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:13.530 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:13.531 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:13.531 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:13.531 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:13.531 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:13.531 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:13.531 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:13.532 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:13.551 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:13.551 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:13.552 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:13.552 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:13.552 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:13.552 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:13.552 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:13.552 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:13.552 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:13.571 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:13.571 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:13.571 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:13.571 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:13.571 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:13.572 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:13.572 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:13.572 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:13.573 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:13.591 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:13.591 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:13.591 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:13.591 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:13.591 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:13.591 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:13.592 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:13.592 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:13.592 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:13.611 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:13.611 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:13.611 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:13.611 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:13.611 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:13.611 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:13.611 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:13.611 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:13.612 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:13.650 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:13.651 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:13.651 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:13.651 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:13.651 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:13.651 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:13.651 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:13.651 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:13.652 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:13.690 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:13.691 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:13.691 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:13.691 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:13.691 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:13.691 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:13.691 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:13.691 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:13.692 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:13.693 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@1407736603(state=PENDING, result=null, callback=(continuation of Deferred@1929764711 after retry RPC after error@1955025941) -> (continuation of Deferred@2075826919 after retry RPC after error@1788552712) -> (continuation of Deferred@934818523 after retry RPC after error@2035417029) -> (continuation of Deferred@1452490666 after retry RPC after error@1703667734) -> (continuation of Deferred@332603542 after retry RPC after error@1703566078) -> (continuation of Deferred@715868042 after retry RPC after error@274383249), errback=(continuation of Deferred@1929764711 after retry RPC after error@1955025941) -> (continuation of Deferred@2075826919 after retry RPC after error@1788552712) -> (continuation of Deferred@934818523 after retry RPC after error@2035417029) -> (continuation of Deferred@1452490666 after retry RPC after error@1703667734) -> (continuation of Deferred@332603542 after retry RPC after error@1703566078) -> (continuation of Deferred@715868042 after retry RPC after error@274383249)))
20:28:13.693 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@1407736603(state=PENDING, result=null, callback=(continuation of Deferred@1929764711 after retry RPC after error@1955025941) -> (continuation of Deferred@2075826919 after retry RPC after error@1788552712) -> (continuation of Deferred@934818523 after retry RPC after error@2035417029) -> (continuation of Deferred@1452490666 after retry RPC after error@1703667734) -> (continuation of Deferred@332603542 after retry RPC after error@1703566078) -> (continuation of Deferred@715868042 after retry RPC after error@274383249), errback=(continuation of Deferred@1929764711 after retry RPC after error@1955025941) -> (continuation of Deferred@2075826919 after retry RPC after error@1788552712) -> (continuation of Deferred@934818523 after retry RPC after error@2035417029) -> (continuation of Deferred@1452490666 after retry RPC after error@1703667734) -> (continuation of Deferred@332603542 after retry RPC after error@1703566078) -> (continuation of Deferred@715868042 after retry RPC after error@274383249)))
20:28:13.710 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:13.711 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:13.711 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:13.711 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:13.711 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:13.711 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:13.711 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:13.711 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:13.712 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:13.730 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:13.730 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:13.731 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:13.731 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:13.731 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:13.731 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:13.731 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:13.731 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:13.732 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:13.751 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:13.751 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:13.752 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:13.752 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:13.752 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:13.752 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:13.752 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:13.752 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:13.752 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:13.771 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:13.771 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:13.771 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:13.772 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:13.772 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:13.772 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:13.772 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:13.772 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:13.772 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:13.772 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:13.772 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:13.772 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:13.772 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:13.773 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:13.773 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:13.790 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:13.791 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:13.791 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:13.791 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:13.791 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:13.791 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:13.791 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:13.791 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:13.792 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:13.830 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:13.831 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:13.831 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:13.831 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:13.831 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:13.831 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:13.831 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:13.831 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:13.832 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:13.890 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:13.891 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:13.891 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:13.891 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:13.891 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:13.891 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:13.891 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:13.891 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:13.892 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:13.951 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:13.951 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:13.951 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:13.951 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:13.951 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:13.951 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:13.951 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:13.951 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:13.952 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:13.952 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=241), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] refreshing cache from master, [62ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [62ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [62ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [63ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@1025455450(state=PENDING, result=null, callback=(continuation of Deferred@2066807633 after retry RPC after error@1897756202) -> (continuation of Deferred@1027937569 after retry RPC after error@2033467958) -> (continuation of Deferred@94553138 after retry RPC after error@521428618) -> (continuation of Deferred@183352404 after retry RPC after error@1368619725) -> (continuation of Deferred@1115928473 after retry RPC after error@2136887629) -> (continuation of Deferred@414600686 after retry RPC after error@1387858215) -> (continuation of Deferred@232346518 after retry RPC after error@8433497), errback=(continuation of Deferred@2066807633 after retry RPC after error@1897756202) -> (continuation of Deferred@1027937569 after retry RPC after error@2033467958) -> (continuation of Deferred@94553138 after retry RPC after error@521428618) -> (continuation of Deferred@183352404 after retry RPC after error@1368619725) -> (continuation of Deferred@1115928473 after retry RPC after error@2136887629) -> (continuation of Deferred@414600686 after retry RPC after error@1387858215) -> (continuation of Deferred@232346518 after retry RPC after error@8433497)))
20:28:13.953 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=241), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] refreshing cache from master, [62ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [62ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [62ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [63ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@1025455450(state=PENDING, result=null, callback=(continuation of Deferred@2066807633 after retry RPC after error@1897756202) -> (continuation of Deferred@1027937569 after retry RPC after error@2033467958) -> (continuation of Deferred@94553138 after retry RPC after error@521428618) -> (continuation of Deferred@183352404 after retry RPC after error@1368619725) -> (continuation of Deferred@1115928473 after retry RPC after error@2136887629) -> (continuation of Deferred@414600686 after retry RPC after error@1387858215) -> (continuation of Deferred@232346518 after retry RPC after error@8433497), errback=(continuation of Deferred@2066807633 after retry RPC after error@1897756202) -> (continuation of Deferred@1027937569 after retry RPC after error@2033467958) -> (continuation of Deferred@94553138 after retry RPC after error@521428618) -> (continuation of Deferred@183352404 after retry RPC after error@1368619725) -> (continuation of Deferred@1115928473 after retry RPC after error@2136887629) -> (continuation of Deferred@414600686 after retry RPC after error@1387858215) -> (continuation of Deferred@232346518 after retry RPC after error@8433497)))
20:28:13.970 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:13.971 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:13.971 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:13.971 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:13.971 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:13.971 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:13.972 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:13.972 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:13.972 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:13.990 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:13.991 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:13.991 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:13.991 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:13.991 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:13.992 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:13.992 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:13.992 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:13.992 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:14.010 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:14.011 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:14.011 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:14.011 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:14.011 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:14.011 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:14.011 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:14.011 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:14.012 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:14.030 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:14.031 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:14.031 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:14.031 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:14.031 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:14.031 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:14.031 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:14.031 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:14.032 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:14.050 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:14.051 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:14.051 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:14.051 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:14.051 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:14.051 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:14.051 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:14.051 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:14.052 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:14.070 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:14.070 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:14.071 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:14.071 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:14.071 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:14.071 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:14.071 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:14.071 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:14.072 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:14.151 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:14.151 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:14.151 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:14.151 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:14.151 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:14.151 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:14.151 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:14.151 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:14.153 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:14.170 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:14.171 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:14.171 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:14.171 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:14.171 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:14.171 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:14.171 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:14.171 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:14.172 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:14.172 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=201), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [183ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@1123747364(state=PENDING, result=null, callback=(continuation of Deferred@1054893606 after retry RPC after error@279480401) -> (continuation of Deferred@1199106521 after retry RPC after error@1571298612) -> (continuation of Deferred@1615102449 after retry RPC after error@880441256) -> (continuation of Deferred@877422303 after retry RPC after error@697343199) -> (continuation of Deferred@1830800149 after retry RPC after error@171914793) -> (continuation of Deferred@1561510703 after retry RPC after error@1883884086) -> (continuation of Deferred@1671295049 after retry RPC after error@274730093), errback=(continuation of Deferred@1054893606 after retry RPC after error@279480401) -> (continuation of Deferred@1199106521 after retry RPC after error@1571298612) -> (continuation of Deferred@1615102449 after retry RPC after error@880441256) -> (continuation of Deferred@877422303 after retry RPC after error@697343199) -> (continuation of Deferred@1830800149 after retry RPC after error@171914793) -> (continuation of Deferred@1561510703 after retry RPC after error@1883884086) -> (continuation of Deferred@1671295049 after retry RPC after error@274730093)))
20:28:14.173 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=201), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [183ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@1123747364(state=PENDING, result=null, callback=(continuation of Deferred@1054893606 after retry RPC after error@279480401) -> (continuation of Deferred@1199106521 after retry RPC after error@1571298612) -> (continuation of Deferred@1615102449 after retry RPC after error@880441256) -> (continuation of Deferred@877422303 after retry RPC after error@697343199) -> (continuation of Deferred@1830800149 after retry RPC after error@171914793) -> (continuation of Deferred@1561510703 after retry RPC after error@1883884086) -> (continuation of Deferred@1671295049 after retry RPC after error@274730093), errback=(continuation of Deferred@1054893606 after retry RPC after error@279480401) -> (continuation of Deferred@1199106521 after retry RPC after error@1571298612) -> (continuation of Deferred@1615102449 after retry RPC after error@880441256) -> (continuation of Deferred@877422303 after retry RPC after error@697343199) -> (continuation of Deferred@1830800149 after retry RPC after error@171914793) -> (continuation of Deferred@1561510703 after retry RPC after error@1883884086) -> (continuation of Deferred@1671295049 after retry RPC after error@274730093)))
20:28:14.190 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:14.191 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:14.191 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:14.191 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:14.191 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:14.191 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:14.191 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:14.191 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:14.192 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:14.210 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:14.210 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:14.211 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:14.211 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:14.211 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:14.211 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:14.211 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:14.211 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:14.211 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:14.231 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:14.231 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:14.232 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:14.232 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:14.232 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:14.232 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:14.232 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:14.232 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:14.233 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:14.251 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:14.251 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:14.251 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:14.251 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:14.251 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:14.252 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:14.252 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:14.252 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:14.252 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:14.271 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:14.271 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:14.271 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:14.271 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:14.271 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:14.272 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:14.272 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:14.272 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:14.272 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:14.290 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:14.291 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:14.291 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:14.291 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:14.291 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:14.291 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:14.291 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:14.291 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:14.292 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:14.350 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:14.351 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:14.351 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:14.351 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:14.351 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:14.351 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:14.351 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:14.351 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:14.352 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:14.450 [DEBUG - pool-40-thread-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=259), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@1215990795(state=PENDING, result=null, callback=(continuation of Deferred@920249367 after retry RPC after error@305777562) -> (continuation of Deferred@910012817 after retry RPC after error@716925321) -> (continuation of Deferred@1396427157 after retry RPC after error@251837811) -> (continuation of Deferred@1424503867 after retry RPC after error@636413966) -> (continuation of Deferred@672459335 after retry RPC after error@854008595) -> (continuation of Deferred@666599576 after retry RPC after error@1976021753) -> (continuation of Deferred@2047752404 after retry RPC after error@1912238909), errback=(continuation of Deferred@920249367 after retry RPC after error@305777562) -> (continuation of Deferred@910012817 after retry RPC after error@716925321) -> (continuation of Deferred@1396427157 after retry RPC after error@251837811) -> (continuation of Deferred@1424503867 after retry RPC after error@636413966) -> (continuation of Deferred@672459335 after retry RPC after error@854008595) -> (continuation of Deferred@666599576 after retry RPC after error@1976021753) -> (continuation of Deferred@2047752404 after retry RPC after error@1912238909)))
20:28:14.451 [WARN - pool-40-thread-1] (HashedWheelTimer.java:716) An exception was thrown by TimerTask.
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713)
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701)
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:787)
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501)
at java.base/java.lang.Thread.run(Thread.java:840)
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713)
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701)
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:787)
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501)
at java.base/java.lang.Thread.run(Thread.java:840)
20:28:14.470 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:14.471 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:14.471 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:14.471 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:14.471 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:14.471 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:14.471 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:14.471 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:14.472 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:14.490 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:14.491 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:14.491 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:14.491 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:14.491 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:14.491 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:14.491 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:14.491 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:14.492 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:14.510 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:14.511 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:14.511 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:14.511 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:14.511 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:14.511 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:14.511 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:14.511 [INFO - pool-40-thread-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:14.512 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:14.531 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:14.531 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:14.532 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:14.532 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:14.532 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:14.532 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:14.532 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:14.532 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:14.532 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:14.551 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:14.551 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:14.552 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:14.552 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:14.552 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:14.552 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:14.552 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:14.552 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:14.553 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:14.571 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:14.571 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:14.572 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:14.572 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:14.572 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:14.572 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:14.572 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:14.572 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:14.572 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:14.631 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:14.631 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:14.632 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:14.632 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:14.632 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:14.632 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:14.632 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:14.632 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:14.632 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:14.633 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [161ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@1960165522(state=PENDING, result=null, callback=(continuation of Deferred@2076884483 after retry RPC after error@1768744625) -> (continuation of Deferred@267687261 after retry RPC after error@1434387809) -> (continuation of Deferred@2070643563 after retry RPC after error@1966910425) -> (continuation of Deferred@1581880926 after retry RPC after error@1932580091) -> (continuation of Deferred@1424449837 after retry RPC after error@1150642699) -> (continuation of Deferred@1459421089 after retry RPC after error@589201033), errback=(continuation of Deferred@2076884483 after retry RPC after error@1768744625) -> (continuation of Deferred@267687261 after retry RPC after error@1434387809) -> (continuation of Deferred@2070643563 after retry RPC after error@1966910425) -> (continuation of Deferred@1581880926 after retry RPC after error@1932580091) -> (continuation of Deferred@1424449837 after retry RPC after error@1150642699) -> (continuation of Deferred@1459421089 after retry RPC after error@589201033)))
20:28:14.633 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [161ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@1960165522(state=PENDING, result=null, callback=(continuation of Deferred@2076884483 after retry RPC after error@1768744625) -> (continuation of Deferred@267687261 after retry RPC after error@1434387809) -> (continuation of Deferred@2070643563 after retry RPC after error@1966910425) -> (continuation of Deferred@1581880926 after retry RPC after error@1932580091) -> (continuation of Deferred@1424449837 after retry RPC after error@1150642699) -> (continuation of Deferred@1459421089 after retry RPC after error@589201033), errback=(continuation of Deferred@2076884483 after retry RPC after error@1768744625) -> (continuation of Deferred@267687261 after retry RPC after error@1434387809) -> (continuation of Deferred@2070643563 after retry RPC after error@1966910425) -> (continuation of Deferred@1581880926 after retry RPC after error@1932580091) -> (continuation of Deferred@1424449837 after retry RPC after error@1150642699) -> (continuation of Deferred@1459421089 after retry RPC after error@589201033)))
20:28:14.651 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:14.651 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:14.651 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:14.651 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:14.651 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:14.652 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:14.652 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:14.652 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:14.652 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:14.671 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:14.671 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:14.671 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:14.671 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:14.672 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:14.672 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:14.672 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:14.672 [INFO - pool-40-thread-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:14.672 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:14.690 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:14.691 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:14.691 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:14.691 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:14.691 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:14.691 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:14.691 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:14.691 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:14.692 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:14.711 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:14.711 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:14.711 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:14.711 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:14.711 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:14.712 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:14.712 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:14.712 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:14.712 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:14.731 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:14.731 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:14.731 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:14.731 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:14.732 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:14.732 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:14.732 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:14.732 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:14.732 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:14.750 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:14.751 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:14.751 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:14.751 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:14.751 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:14.751 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:14.751 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:14.751 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:14.752 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:14.790 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:14.790 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:14.791 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:14.791 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:14.791 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:14.791 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:14.791 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:14.791 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:14.792 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:14.870 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:14.871 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:14.871 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:14.871 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:14.871 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:14.871 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:14.871 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:14.871 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:14.872 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:14.872 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=221), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [0ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [1ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [20ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [40ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [60ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [99ms] refreshing cache from master, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [100ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [100ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [101ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [139ms] refreshing cache from master, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [140ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [140ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [141ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [219ms] refreshing cache from master, [219ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [219ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [220ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [220ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [221ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@1929268191(state=PENDING, result=null, callback=(continuation of Deferred@2061019303 after retry RPC after error@1156122889) -> (continuation of Deferred@2081637416 after retry RPC after error@2083739846) -> (continuation of Deferred@1133595255 after retry RPC after error@1337766477) -> (continuation of Deferred@694717846 after retry RPC after error@1341850901) -> (continuation of Deferred@108014703 after retry RPC after error@754239542) -> (continuation of Deferred@1092482051 after retry RPC after error@1457099416) -> (continuation of Deferred@1372442488 after retry RPC after error@1542592527), errback=(continuation of Deferred@2061019303 after retry RPC after error@1156122889) -> (continuation of Deferred@2081637416 after retry RPC after error@2083739846) -> (continuation of Deferred@1133595255 after retry RPC after error@1337766477) -> (continuation of Deferred@694717846 after retry RPC after error@1341850901) -> (continuation of Deferred@108014703 after retry RPC after error@754239542) -> (continuation of Deferred@1092482051 after retry RPC after error@1457099416) -> (continuation of Deferred@1372442488 after retry RPC after error@1542592527)))
20:28:14.873 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=221), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [0ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [1ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [20ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [40ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [60ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [99ms] refreshing cache from master, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [100ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [100ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [101ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [139ms] refreshing cache from master, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [140ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [140ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [141ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [219ms] refreshing cache from master, [219ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [219ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [220ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [220ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [221ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@1929268191(state=PENDING, result=null, callback=(continuation of Deferred@2061019303 after retry RPC after error@1156122889) -> (continuation of Deferred@2081637416 after retry RPC after error@2083739846) -> (continuation of Deferred@1133595255 after retry RPC after error@1337766477) -> (continuation of Deferred@694717846 after retry RPC after error@1341850901) -> (continuation of Deferred@108014703 after retry RPC after error@754239542) -> (continuation of Deferred@1092482051 after retry RPC after error@1457099416) -> (continuation of Deferred@1372442488 after retry RPC after error@1542592527), errback=(continuation of Deferred@2061019303 after retry RPC after error@1156122889) -> (continuation of Deferred@2081637416 after retry RPC after error@2083739846) -> (continuation of Deferred@1133595255 after retry RPC after error@1337766477) -> (continuation of Deferred@694717846 after retry RPC after error@1341850901) -> (continuation of Deferred@108014703 after retry RPC after error@754239542) -> (continuation of Deferred@1092482051 after retry RPC after error@1457099416) -> (continuation of Deferred@1372442488 after retry RPC after error@1542592527)))
20:28:14.891 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:14.891 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:14.891 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:14.891 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:14.891 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:14.891 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:14.891 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:14.891 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:14.893 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:14.910 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:14.911 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:14.911 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:14.911 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:14.911 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:14.911 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:14.912 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:14.912 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:14.912 [WARN - kudu-nio-6] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:14.930 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:14.931 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:14.931 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:14.931 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:14.931 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:14.931 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:14.931 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:14.931 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:14.933 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:14.950 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:14.950 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:14.951 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:14.951 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:14.951 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:14.951 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:14.951 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:14.951 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:14.952 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:14.971 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:14.971 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:14.972 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:14.972 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:14.972 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:14.972 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:14.972 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:14.972 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:14.972 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:14.991 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:14.991 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:14.991 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:14.991 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:14.991 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:14.991 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:14.992 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:14.992 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:14.992 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:15.051 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:15.051 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:15.052 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:15.052 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:15.052 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:15.052 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:15.052 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:15.052 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:15.053 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:15.091 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:15.091 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:15.092 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:15.092 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:15.092 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:15.092 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:15.092 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:15.092 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:15.092 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:15.131 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:15.131 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:15.132 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:15.132 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:15.132 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:15.132 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:15.132 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:15.132 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:15.133 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:15.133 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [3ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [161ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [201ms] refreshing cache from master, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [203ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [241ms] refreshing cache from master, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [243ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [243ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@968383911(state=PENDING, result=null, callback=(continuation of Deferred@378790230 after retry RPC after error@939780642) -> (continuation of Deferred@1066576052 after retry RPC after error@1153208196) -> (continuation of Deferred@1294753556 after retry RPC after error@1792667241) -> (continuation of Deferred@1076116622 after retry RPC after error@1623481249) -> (continuation of Deferred@1337949266 after retry RPC after error@1789946218) -> (continuation of Deferred@185723134 after retry RPC after error@1229628270) -> (continuation of Deferred@2099452921 after retry RPC after error@338810382) -> (continuation of Deferred@283347566 after retry RPC after error@212732623), errback=(continuation of Deferred@378790230 after retry RPC after error@939780642) -> (continuation of Deferred@1066576052 after retry RPC after error@1153208196) -> (continuation of Deferred@1294753556 after retry RPC after error@1792667241) -> (continuation of Deferred@1076116622 after retry RPC after error@1623481249) -> (continuation of Deferred@1337949266 after retry RPC after error@1789946218) -> (continuation of Deferred@185723134 after retry RPC after error@1229628270) -> (continuation of Deferred@2099452921 after retry RPC after error@338810382) -> (continuation of Deferred@283347566 after retry RPC after error@212732623)))
20:28:15.133 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [3ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [161ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [201ms] refreshing cache from master, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [203ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [241ms] refreshing cache from master, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [243ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [243ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@968383911(state=PENDING, result=null, callback=(continuation of Deferred@378790230 after retry RPC after error@939780642) -> (continuation of Deferred@1066576052 after retry RPC after error@1153208196) -> (continuation of Deferred@1294753556 after retry RPC after error@1792667241) -> (continuation of Deferred@1076116622 after retry RPC after error@1623481249) -> (continuation of Deferred@1337949266 after retry RPC after error@1789946218) -> (continuation of Deferred@185723134 after retry RPC after error@1229628270) -> (continuation of Deferred@2099452921 after retry RPC after error@338810382) -> (continuation of Deferred@283347566 after retry RPC after error@212732623), errback=(continuation of Deferred@378790230 after retry RPC after error@939780642) -> (continuation of Deferred@1066576052 after retry RPC after error@1153208196) -> (continuation of Deferred@1294753556 after retry RPC after error@1792667241) -> (continuation of Deferred@1076116622 after retry RPC after error@1623481249) -> (continuation of Deferred@1337949266 after retry RPC after error@1789946218) -> (continuation of Deferred@185723134 after retry RPC after error@1229628270) -> (continuation of Deferred@2099452921 after retry RPC after error@338810382) -> (continuation of Deferred@283347566 after retry RPC after error@212732623)))
20:28:15.151 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:15.151 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:15.152 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:15.152 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:15.152 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:15.152 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:15.152 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:15.152 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:15.152 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:15.171 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:15.171 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:15.171 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:15.171 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:15.171 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:15.171 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:15.172 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:15.172 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:15.172 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:15.191 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:15.191 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:15.191 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:15.191 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:15.191 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:15.191 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:15.191 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:15.191 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:15.192 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:15.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:28:15.205022 4132 master_runner.cc:186] Error getting master registration for 127.3.46.61:39407: Network error: Client connection negotiation failed: client connection to 127.3.46.61:39407: connect: Connection refused (error 111),
20:28:15.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:28:15.205204 4132 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
20:28:15.210 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:15.211 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:15.211 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:15.211 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:15.211 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:15.211 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:15.211 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:15.211 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:15.212 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:15.230 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:15.231 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:15.231 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:15.231 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:15.231 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:15.231 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:15.231 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:15.231 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:15.232 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:15.250 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:15.250 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:15.251 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:15.251 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:15.251 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:15.251 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:15.251 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:15.251 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:15.252 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:15.290 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:15.290 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:15.291 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:15.291 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:15.291 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:15.291 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:15.291 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:15.291 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:15.292 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:15.292 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=140), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [20ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [40ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [60ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [80ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [99ms] refreshing cache from master, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [100ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [100ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [101ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [139ms] refreshing cache from master, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [140ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [140ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [140ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [141ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@1397784550(state=PENDING, result=null, callback=(continuation of Deferred@1083719820 after retry RPC after error@1025878578) -> (continuation of Deferred@409898009 after retry RPC after error@1217494973) -> (continuation of Deferred@1242552835 after retry RPC after error@1516997962) -> (continuation of Deferred@915004506 after retry RPC after error@1973821857) -> (continuation of Deferred@1103966932 after retry RPC after error@1196095098) -> (continuation of Deferred@1062836677 after retry RPC after error@2075532454), errback=(continuation of Deferred@1083719820 after retry RPC after error@1025878578) -> (continuation of Deferred@409898009 after retry RPC after error@1217494973) -> (continuation of Deferred@1242552835 after retry RPC after error@1516997962) -> (continuation of Deferred@915004506 after retry RPC after error@1973821857) -> (continuation of Deferred@1103966932 after retry RPC after error@1196095098) -> (continuation of Deferred@1062836677 after retry RPC after error@2075532454)))
20:28:15.292 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=140), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [20ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [40ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [60ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [80ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [99ms] refreshing cache from master, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [100ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [100ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [101ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [139ms] refreshing cache from master, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [140ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [140ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [140ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [141ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@1397784550(state=PENDING, result=null, callback=(continuation of Deferred@1083719820 after retry RPC after error@1025878578) -> (continuation of Deferred@409898009 after retry RPC after error@1217494973) -> (continuation of Deferred@1242552835 after retry RPC after error@1516997962) -> (continuation of Deferred@915004506 after retry RPC after error@1973821857) -> (continuation of Deferred@1103966932 after retry RPC after error@1196095098) -> (continuation of Deferred@1062836677 after retry RPC after error@2075532454), errback=(continuation of Deferred@1083719820 after retry RPC after error@1025878578) -> (continuation of Deferred@409898009 after retry RPC after error@1217494973) -> (continuation of Deferred@1242552835 after retry RPC after error@1516997962) -> (continuation of Deferred@915004506 after retry RPC after error@1973821857) -> (continuation of Deferred@1103966932 after retry RPC after error@1196095098) -> (continuation of Deferred@1062836677 after retry RPC after error@2075532454)))
20:28:15.311 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:15.311 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:15.312 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:15.312 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:15.312 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:15.312 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:15.312 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:15.312 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:15.313 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:15.331 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:15.331 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:15.332 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:15.332 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:15.332 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:15.332 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:15.332 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:15.332 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:15.333 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:15.351 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:15.351 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:15.352 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:15.352 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:15.352 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:15.352 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:15.352 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:15.352 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:15.353 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:15.371 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:15.371 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:15.372 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:15.372 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:15.372 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:15.372 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:15.372 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:15.372 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:15.373 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:15.391 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:15.391 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:15.392 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:15.392 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:15.392 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:15.392 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:15.392 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:15.392 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:15.392 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:15.410 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:15.411 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:15.411 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:15.411 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:15.411 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:15.411 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:15.411 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:15.411 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:15.412 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:15.470 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:15.471 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:15.471 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:15.471 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:15.471 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:15.471 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:15.471 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:15.471 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:15.472 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:15.510 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:15.511 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:15.511 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:15.511 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:15.511 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:15.511 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:15.511 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:15.511 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:15.512 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:15.550 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:15.550 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:15.551 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:15.551 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:15.551 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:15.551 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:15.551 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:15.551 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:15.552 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:15.552 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=241), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [99ms] refreshing cache from master, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [100ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [100ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [101ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [159ms] refreshing cache from master, [159ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [159ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [160ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [160ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [161ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [199ms] refreshing cache from master, [199ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [199ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [200ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [200ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [201ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [239ms] refreshing cache from master, [239ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [239ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [239ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [240ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [240ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [241ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, deferred=Deferred@104643750(state=PENDING, result=null, callback=(continuation of Deferred@634315468 after retry RPC after error@1417284335) -> (continuation of Deferred@764565896 after retry RPC after error@1319639312) -> (continuation of Deferred@480397963 after retry RPC after error@411764354) -> (continuation of Deferred@669667842 after retry RPC after error@696565622) -> (continuation of Deferred@670846687 after retry RPC after error@1736696756) -> (continuation of Deferred@302827268 after retry RPC after error@520074556) -> (continuation of Deferred@1538300554 after retry RPC after error@601246731) -> (continuation of Deferred@1469483922 after retry RPC after error@883134985), errback=(continuation of Deferred@634315468 after retry RPC after error@1417284335) -> (continuation of Deferred@764565896 after retry RPC after error@1319639312) -> (continuation of Deferred@480397963 after retry RPC after error@411764354) -> (continuation of Deferred@669667842 after retry RPC after error@696565622) -> (continuation of Deferred@670846687 after retry RPC after error@1736696756) -> (continuation of Deferred@302827268 after retry RPC after error@520074556) -> (continuation of Deferred@1538300554 after retry RPC after error@601246731) -> (continuation of Deferred@1469483922 after retry RPC after error@883134985)))
20:28:15.552 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=241), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [99ms] refreshing cache from master, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [100ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [100ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [101ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [159ms] refreshing cache from master, [159ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [159ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [160ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [160ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [161ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [199ms] refreshing cache from master, [199ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [199ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [200ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [200ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [201ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [239ms] refreshing cache from master, [239ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [239ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [239ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [240ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [240ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [241ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, deferred=Deferred@104643750(state=PENDING, result=null, callback=(continuation of Deferred@634315468 after retry RPC after error@1417284335) -> (continuation of Deferred@764565896 after retry RPC after error@1319639312) -> (continuation of Deferred@480397963 after retry RPC after error@411764354) -> (continuation of Deferred@669667842 after retry RPC after error@696565622) -> (continuation of Deferred@670846687 after retry RPC after error@1736696756) -> (continuation of Deferred@302827268 after retry RPC after error@520074556) -> (continuation of Deferred@1538300554 after retry RPC after error@601246731) -> (continuation of Deferred@1469483922 after retry RPC after error@883134985), errback=(continuation of Deferred@634315468 after retry RPC after error@1417284335) -> (continuation of Deferred@764565896 after retry RPC after error@1319639312) -> (continuation of Deferred@480397963 after retry RPC after error@411764354) -> (continuation of Deferred@669667842 after retry RPC after error@696565622) -> (continuation of Deferred@670846687 after retry RPC after error@1736696756) -> (continuation of Deferred@302827268 after retry RPC after error@520074556) -> (continuation of Deferred@1538300554 after retry RPC after error@601246731) -> (continuation of Deferred@1469483922 after retry RPC after error@883134985)))
20:28:15.571 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:15.571 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:15.572 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:15.572 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:15.572 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:15.572 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:15.572 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:15.572 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:15.573 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:15.591 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:15.591 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:15.591 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:15.591 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:15.592 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:15.592 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:15.592 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:15.592 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:15.592 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:15.611 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:15.611 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:15.611 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:15.611 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:15.611 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:15.611 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:15.611 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:15.611 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:15.612 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:15.630 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:15.631 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:15.631 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:15.631 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:15.631 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:15.631 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:15.631 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:15.631 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:15.632 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:15.650 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:15.650 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:15.651 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:15.651 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:15.651 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:15.651 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:15.651 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:15.651 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:15.652 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:15.671 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:15.671 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:15.672 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:15.672 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:15.672 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:15.672 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:15.672 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:15.672 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:15.672 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:15.731 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:15.731 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:15.732 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:15.732 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:15.732 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:15.732 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:15.732 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:15.732 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:15.733 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:15.733 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=161), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [40ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [60ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [80ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, deferred=Deferred@459877280(state=PENDING, result=null, callback=(continuation of Deferred@494745569 after retry RPC after error@907758226) -> (continuation of Deferred@1791396894 after retry RPC after error@950046413) -> (continuation of Deferred@1276535815 after retry RPC after error@548792525) -> (continuation of Deferred@2127378105 after retry RPC after error@1118247243) -> (continuation of Deferred@1039039300 after retry RPC after error@848433076) -> (continuation of Deferred@628989857 after retry RPC after error@1600829188), errback=(continuation of Deferred@494745569 after retry RPC after error@907758226) -> (continuation of Deferred@1791396894 after retry RPC after error@950046413) -> (continuation of Deferred@1276535815 after retry RPC after error@548792525) -> (continuation of Deferred@2127378105 after retry RPC after error@1118247243) -> (continuation of Deferred@1039039300 after retry RPC after error@848433076) -> (continuation of Deferred@628989857 after retry RPC after error@1600829188)))
20:28:15.733 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=161), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [40ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [60ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [80ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, deferred=Deferred@459877280(state=PENDING, result=null, callback=(continuation of Deferred@494745569 after retry RPC after error@907758226) -> (continuation of Deferred@1791396894 after retry RPC after error@950046413) -> (continuation of Deferred@1276535815 after retry RPC after error@548792525) -> (continuation of Deferred@2127378105 after retry RPC after error@1118247243) -> (continuation of Deferred@1039039300 after retry RPC after error@848433076) -> (continuation of Deferred@628989857 after retry RPC after error@1600829188), errback=(continuation of Deferred@494745569 after retry RPC after error@907758226) -> (continuation of Deferred@1791396894 after retry RPC after error@950046413) -> (continuation of Deferred@1276535815 after retry RPC after error@548792525) -> (continuation of Deferred@2127378105 after retry RPC after error@1118247243) -> (continuation of Deferred@1039039300 after retry RPC after error@848433076) -> (continuation of Deferred@628989857 after retry RPC after error@1600829188)))
20:28:15.751 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:15.751 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:15.751 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:15.751 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:15.751 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:15.751 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:15.751 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:15.751 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:15.752 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:15.770 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:15.771 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:15.771 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:15.771 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:15.771 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:15.771 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:15.772 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:15.772 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:15.772 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:15.790 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:15.791 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:15.791 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:15.791 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:15.791 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:15.791 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:15.791 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:15.791 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:15.792 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:15.810 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:15.811 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:15.811 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:15.811 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:15.811 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:15.811 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:15.811 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:15.811 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:15.812 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:15.830 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:15.830 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:15.831 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:15.831 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:15.831 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:15.831 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:15.831 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:15.831 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:15.832 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:15.871 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:15.871 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:15.872 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:15.872 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:15.872 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:15.872 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:15.872 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:15.872 [INFO - pool-40-thread-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:15.872 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:15.890 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:15.891 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:15.891 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:15.891 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:15.891 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:15.891 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:15.891 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:15.891 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:15.892 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:15.991 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:15.991 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:15.991 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:15.992 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:15.992 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:15.992 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:15.992 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:15.992 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:15.994 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:15.994 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=243), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [241ms] refreshing cache from master, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [244ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [244ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@1733568735(state=PENDING, result=null, callback=(continuation of Deferred@953720339 after retry RPC after error@1712752922) -> (continuation of Deferred@1881065817 after retry RPC after error@919209301) -> (continuation of Deferred@1251884441 after retry RPC after error@412230537) -> (continuation of Deferred@507536835 after retry RPC after error@2017864691) -> (continuation of Deferred@1125102487 after retry RPC after error@1250446868) -> (continuation of Deferred@1476084987 after retry RPC after error@506197060) -> (continuation of Deferred@537292573 after retry RPC after error@678850023), errback=(continuation of Deferred@953720339 after retry RPC after error@1712752922) -> (continuation of Deferred@1881065817 after retry RPC after error@919209301) -> (continuation of Deferred@1251884441 after retry RPC after error@412230537) -> (continuation of Deferred@507536835 after retry RPC after error@2017864691) -> (continuation of Deferred@1125102487 after retry RPC after error@1250446868) -> (continuation of Deferred@1476084987 after retry RPC after error@506197060) -> (continuation of Deferred@537292573 after retry RPC after error@678850023)))
20:28:15.995 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=243), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [241ms] refreshing cache from master, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [244ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [244ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@1733568735(state=PENDING, result=null, callback=(continuation of Deferred@953720339 after retry RPC after error@1712752922) -> (continuation of Deferred@1881065817 after retry RPC after error@919209301) -> (continuation of Deferred@1251884441 after retry RPC after error@412230537) -> (continuation of Deferred@507536835 after retry RPC after error@2017864691) -> (continuation of Deferred@1125102487 after retry RPC after error@1250446868) -> (continuation of Deferred@1476084987 after retry RPC after error@506197060) -> (continuation of Deferred@537292573 after retry RPC after error@678850023), errback=(continuation of Deferred@953720339 after retry RPC after error@1712752922) -> (continuation of Deferred@1881065817 after retry RPC after error@919209301) -> (continuation of Deferred@1251884441 after retry RPC after error@412230537) -> (continuation of Deferred@507536835 after retry RPC after error@2017864691) -> (continuation of Deferred@1125102487 after retry RPC after error@1250446868) -> (continuation of Deferred@1476084987 after retry RPC after error@506197060) -> (continuation of Deferred@537292573 after retry RPC after error@678850023)))
20:28:16.011 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:16.011 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:16.012 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:16.012 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:16.012 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:16.012 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:16.012 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:16.012 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:16.013 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:16.031 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:16.032 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:16.032 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:16.032 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:16.032 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:16.032 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:16.032 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:16.032 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:16.033 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:16.050 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:16.050 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:16.051 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:16.051 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:16.051 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:16.051 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:16.051 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:16.051 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:16.052 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:16.071 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:16.071 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:16.071 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:16.071 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:16.072 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:16.072 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:16.072 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:16.072 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:16.072 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:16.091 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:16.091 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:16.091 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:16.091 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:16.091 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:16.091 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:16.091 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:16.091 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:16.092 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:16.110 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:16.111 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:16.111 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:16.111 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:16.111 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:16.111 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:16.112 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:16.112 [INFO - pool-40-thread-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:16.112 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:16.170 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:16.171 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:16.171 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:16.171 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:16.171 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:16.171 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:16.172 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:16.172 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:16.172 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:16.190 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:16.191 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:16.191 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:16.191 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:16.191 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:16.191 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:16.191 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:16.191 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:16.192 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:16.192 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=181), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [40ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [80ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [99ms] refreshing cache from master, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [100ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [101ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [159ms] refreshing cache from master, [159ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [159ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [160ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [161ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [179ms] refreshing cache from master, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [180ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [180ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [181ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@1895784617(state=PENDING, result=null, callback=(continuation of Deferred@1862446397 after retry RPC after error@205941516) -> (continuation of Deferred@1721492309 after retry RPC after error@1844063495) -> (continuation of Deferred@402329282 after retry RPC after error@676993338) -> (continuation of Deferred@300074543 after retry RPC after error@761021185) -> (continuation of Deferred@1646183798 after retry RPC after error@452301766) -> (continuation of Deferred@497155337 after retry RPC after error@1386930679) -> (continuation of Deferred@1175174049 after retry RPC after error@1893104389), errback=(continuation of Deferred@1862446397 after retry RPC after error@205941516) -> (continuation of Deferred@1721492309 after retry RPC after error@1844063495) -> (continuation of Deferred@402329282 after retry RPC after error@676993338) -> (continuation of Deferred@300074543 after retry RPC after error@761021185) -> (continuation of Deferred@1646183798 after retry RPC after error@452301766) -> (continuation of Deferred@497155337 after retry RPC after error@1386930679) -> (continuation of Deferred@1175174049 after retry RPC after error@1893104389)))
20:28:16.193 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=181), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [40ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [80ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [99ms] refreshing cache from master, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [100ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [101ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [159ms] refreshing cache from master, [159ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [159ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [160ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [161ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [179ms] refreshing cache from master, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [180ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [180ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [181ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@1895784617(state=PENDING, result=null, callback=(continuation of Deferred@1862446397 after retry RPC after error@205941516) -> (continuation of Deferred@1721492309 after retry RPC after error@1844063495) -> (continuation of Deferred@402329282 after retry RPC after error@676993338) -> (continuation of Deferred@300074543 after retry RPC after error@761021185) -> (continuation of Deferred@1646183798 after retry RPC after error@452301766) -> (continuation of Deferred@497155337 after retry RPC after error@1386930679) -> (continuation of Deferred@1175174049 after retry RPC after error@1893104389), errback=(continuation of Deferred@1862446397 after retry RPC after error@205941516) -> (continuation of Deferred@1721492309 after retry RPC after error@1844063495) -> (continuation of Deferred@402329282 after retry RPC after error@676993338) -> (continuation of Deferred@300074543 after retry RPC after error@761021185) -> (continuation of Deferred@1646183798 after retry RPC after error@452301766) -> (continuation of Deferred@497155337 after retry RPC after error@1386930679) -> (continuation of Deferred@1175174049 after retry RPC after error@1893104389)))
20:28:16.210 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:16.210 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:16.211 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:16.211 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:16.211 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:16.211 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:16.211 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:16.211 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:16.212 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:16.231 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:16.231 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:16.231 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:16.231 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:16.232 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:16.232 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:16.232 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:16.232 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:16.233 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:16.251 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:16.251 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:16.251 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:16.251 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:16.251 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:16.251 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:16.251 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:16.251 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:16.252 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:16.270 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:16.271 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:16.271 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:16.271 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:16.271 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:16.271 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:16.271 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:16.271 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:16.272 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:16.290 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:16.290 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:16.291 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:16.291 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:16.291 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:16.291 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:16.291 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:16.291 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:16.292 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:16.330 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:16.330 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:16.331 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:16.331 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:16.331 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:16.331 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:16.331 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:16.331 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:16.331 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:16.390 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:16.390 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:16.391 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:16.391 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:16.391 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:16.391 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:16.392 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:16.392 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:16.392 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:16.430 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:16.431 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:16.431 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:16.431 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:16.431 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:16.431 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:16.431 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:16.431 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:16.432 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:16.432 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [23ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [182ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [222ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@828572838(state=PENDING, result=null, callback=(continuation of Deferred@1032355023 after retry RPC after error@1091687949) -> (continuation of Deferred@351939204 after retry RPC after error@1024809609) -> (continuation of Deferred@1124411360 after retry RPC after error@216735795) -> (continuation of Deferred@1233163552 after retry RPC after error@1582714763) -> (continuation of Deferred@881219974 after retry RPC after error@1639521693) -> (continuation of Deferred@752725279 after retry RPC after error@1075549464) -> (continuation of Deferred@1188540148 after retry RPC after error@1865090630), errback=(continuation of Deferred@1032355023 after retry RPC after error@1091687949) -> (continuation of Deferred@351939204 after retry RPC after error@1024809609) -> (continuation of Deferred@1124411360 after retry RPC after error@216735795) -> (continuation of Deferred@1233163552 after retry RPC after error@1582714763) -> (continuation of Deferred@881219974 after retry RPC after error@1639521693) -> (continuation of Deferred@752725279 after retry RPC after error@1075549464) -> (continuation of Deferred@1188540148 after retry RPC after error@1865090630)))
20:28:16.432 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [23ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [182ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [222ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@828572838(state=PENDING, result=null, callback=(continuation of Deferred@1032355023 after retry RPC after error@1091687949) -> (continuation of Deferred@351939204 after retry RPC after error@1024809609) -> (continuation of Deferred@1124411360 after retry RPC after error@216735795) -> (continuation of Deferred@1233163552 after retry RPC after error@1582714763) -> (continuation of Deferred@881219974 after retry RPC after error@1639521693) -> (continuation of Deferred@752725279 after retry RPC after error@1075549464) -> (continuation of Deferred@1188540148 after retry RPC after error@1865090630), errback=(continuation of Deferred@1032355023 after retry RPC after error@1091687949) -> (continuation of Deferred@351939204 after retry RPC after error@1024809609) -> (continuation of Deferred@1124411360 after retry RPC after error@216735795) -> (continuation of Deferred@1233163552 after retry RPC after error@1582714763) -> (continuation of Deferred@881219974 after retry RPC after error@1639521693) -> (continuation of Deferred@752725279 after retry RPC after error@1075549464) -> (continuation of Deferred@1188540148 after retry RPC after error@1865090630)))
20:28:16.450 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:16.451 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:16.451 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:16.451 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:16.451 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:16.451 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:16.451 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:16.451 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:16.452 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:16.470 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:16.471 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:16.471 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:16.471 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:16.471 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:16.471 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:16.471 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:16.471 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:16.472 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:16.490 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:16.490 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:16.491 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:16.491 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:16.491 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:16.491 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:16.491 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:16.491 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:16.491 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:16.511 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:16.511 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:16.512 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:16.512 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:16.512 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:16.512 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:16.512 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:16.512 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:16.512 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:16.531 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:16.531 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:16.531 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:16.531 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:16.531 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:16.531 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:16.531 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:16.531 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:16.532 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:16.571 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:16.571 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:16.572 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:16.572 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:16.572 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:16.572 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:16.572 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:16.572 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:16.573 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:16.591 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:16.591 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:16.591 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:16.591 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:16.592 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:16.592 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:16.592 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:16.592 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:16.592 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:16.593 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=142), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [123ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [141ms] refreshing cache from master, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@794081812(state=PENDING, result=null, callback=(continuation of Deferred@500563185 after retry RPC after error@158087784) -> (continuation of Deferred@1527770173 after retry RPC after error@1367435602) -> (continuation of Deferred@1661023719 after retry RPC after error@190250133) -> (continuation of Deferred@798787526 after retry RPC after error@1972220062) -> (continuation of Deferred@524881349 after retry RPC after error@600307949) -> (continuation of Deferred@1756297700 after retry RPC after error@1883225391), errback=(continuation of Deferred@500563185 after retry RPC after error@158087784) -> (continuation of Deferred@1527770173 after retry RPC after error@1367435602) -> (continuation of Deferred@1661023719 after retry RPC after error@190250133) -> (continuation of Deferred@798787526 after retry RPC after error@1972220062) -> (continuation of Deferred@524881349 after retry RPC after error@600307949) -> (continuation of Deferred@1756297700 after retry RPC after error@1883225391)))
20:28:16.593 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=142), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [123ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [141ms] refreshing cache from master, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@794081812(state=PENDING, result=null, callback=(continuation of Deferred@500563185 after retry RPC after error@158087784) -> (continuation of Deferred@1527770173 after retry RPC after error@1367435602) -> (continuation of Deferred@1661023719 after retry RPC after error@190250133) -> (continuation of Deferred@798787526 after retry RPC after error@1972220062) -> (continuation of Deferred@524881349 after retry RPC after error@600307949) -> (continuation of Deferred@1756297700 after retry RPC after error@1883225391), errback=(continuation of Deferred@500563185 after retry RPC after error@158087784) -> (continuation of Deferred@1527770173 after retry RPC after error@1367435602) -> (continuation of Deferred@1661023719 after retry RPC after error@190250133) -> (continuation of Deferred@798787526 after retry RPC after error@1972220062) -> (continuation of Deferred@524881349 after retry RPC after error@600307949) -> (continuation of Deferred@1756297700 after retry RPC after error@1883225391)))
20:28:16.611 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:16.611 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:16.611 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:16.611 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:16.611 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:16.611 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:16.611 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:16.611 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:16.612 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:16.630 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:16.631 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:16.631 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:16.631 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:16.631 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:16.631 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:16.631 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:16.631 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:16.632 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:16.650 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:16.651 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:16.651 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:16.651 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:16.651 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:16.651 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:16.651 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:16.651 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:16.652 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:16.670 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:16.670 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:16.671 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:16.671 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:16.671 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:16.671 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:16.671 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:16.671 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:16.672 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:16.691 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:16.691 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:16.692 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:16.692 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:16.692 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:16.692 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:16.692 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:16.692 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:16.693 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:16.711 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:16.711 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:16.711 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:16.712 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:16.712 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:16.712 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:16.712 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:16.712 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:16.713 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:16.751 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:16.751 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:16.751 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:16.752 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:16.752 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:16.752 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:16.752 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:16.752 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:16.752 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:16.791 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:16.791 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:16.791 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:16.791 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:16.791 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:16.792 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:16.792 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:16.792 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:16.793 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:16.793 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [83ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [103ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [180ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [183ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@1936566371(state=PENDING, result=null, callback=(continuation of Deferred@1370888727 after retry RPC after error@428064632) -> (continuation of Deferred@75734017 after retry RPC after error@439550530) -> (continuation of Deferred@2067060145 after retry RPC after error@2135477712) -> (continuation of Deferred@1731405228 after retry RPC after error@1695549978) -> (continuation of Deferred@953471572 after retry RPC after error@21737171) -> (continuation of Deferred@2133587219 after retry RPC after error@437149750), errback=(continuation of Deferred@1370888727 after retry RPC after error@428064632) -> (continuation of Deferred@75734017 after retry RPC after error@439550530) -> (continuation of Deferred@2067060145 after retry RPC after error@2135477712) -> (continuation of Deferred@1731405228 after retry RPC after error@1695549978) -> (continuation of Deferred@953471572 after retry RPC after error@21737171) -> (continuation of Deferred@2133587219 after retry RPC after error@437149750)))
20:28:16.793 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [83ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [103ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [180ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [183ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@1936566371(state=PENDING, result=null, callback=(continuation of Deferred@1370888727 after retry RPC after error@428064632) -> (continuation of Deferred@75734017 after retry RPC after error@439550530) -> (continuation of Deferred@2067060145 after retry RPC after error@2135477712) -> (continuation of Deferred@1731405228 after retry RPC after error@1695549978) -> (continuation of Deferred@953471572 after retry RPC after error@21737171) -> (continuation of Deferred@2133587219 after retry RPC after error@437149750), errback=(continuation of Deferred@1370888727 after retry RPC after error@428064632) -> (continuation of Deferred@75734017 after retry RPC after error@439550530) -> (continuation of Deferred@2067060145 after retry RPC after error@2135477712) -> (continuation of Deferred@1731405228 after retry RPC after error@1695549978) -> (continuation of Deferred@953471572 after retry RPC after error@21737171) -> (continuation of Deferred@2133587219 after retry RPC after error@437149750)))
20:28:16.811 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:16.811 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:16.812 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:16.812 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:16.812 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:16.812 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:16.812 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:16.812 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:16.813 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:16.831 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:16.831 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:16.832 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:16.832 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:16.832 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:16.832 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:16.832 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:16.832 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:16.833 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:16.851 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:16.851 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:16.851 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:16.851 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:16.851 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:16.852 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:16.852 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:16.852 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:16.852 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:16.871 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:16.871 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:16.871 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:16.871 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:16.871 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:16.872 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:16.872 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:16.872 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:16.872 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:16.890 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:16.891 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:16.891 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:16.891 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:16.891 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:16.891 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:16.891 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:16.891 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:16.892 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:16.930 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:16.931 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:16.931 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:16.931 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:16.931 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:16.931 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:16.931 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:16.931 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:16.932 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:16.970 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:16.971 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:16.971 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:16.971 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:16.971 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:16.971 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:16.971 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:16.971 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:16.972 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:17.010 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:17.011 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:17.011 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:17.011 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:17.011 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:17.012 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:17.012 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:17.012 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:17.012 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:17.050 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:17.051 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:17.051 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:17.051 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:17.051 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:17.051 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:17.051 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:17.051 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:17.052 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:17.053 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [3ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, deferred=Deferred@1167865445(state=PENDING, result=null, callback=(continuation of Deferred@404744473 after retry RPC after error@438484049) -> (continuation of Deferred@2142216031 after retry RPC after error@1529131473) -> (continuation of Deferred@1007996045 after retry RPC after error@870006634) -> (continuation of Deferred@806161193 after retry RPC after error@1322880704) -> (continuation of Deferred@1412645682 after retry RPC after error@1056507569) -> (continuation of Deferred@1718945284 after retry RPC after error@1691888516) -> (continuation of Deferred@708632318 after retry RPC after error@1552320902) -> (continuation of Deferred@302903604 after retry RPC after error@925662458), errback=(continuation of Deferred@404744473 after retry RPC after error@438484049) -> (continuation of Deferred@2142216031 after retry RPC after error@1529131473) -> (continuation of Deferred@1007996045 after retry RPC after error@870006634) -> (continuation of Deferred@806161193 after retry RPC after error@1322880704) -> (continuation of Deferred@1412645682 after retry RPC after error@1056507569) -> (continuation of Deferred@1718945284 after retry RPC after error@1691888516) -> (continuation of Deferred@708632318 after retry RPC after error@1552320902) -> (continuation of Deferred@302903604 after retry RPC after error@925662458)))
20:28:17.053 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [3ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, deferred=Deferred@1167865445(state=PENDING, result=null, callback=(continuation of Deferred@404744473 after retry RPC after error@438484049) -> (continuation of Deferred@2142216031 after retry RPC after error@1529131473) -> (continuation of Deferred@1007996045 after retry RPC after error@870006634) -> (continuation of Deferred@806161193 after retry RPC after error@1322880704) -> (continuation of Deferred@1412645682 after retry RPC after error@1056507569) -> (continuation of Deferred@1718945284 after retry RPC after error@1691888516) -> (continuation of Deferred@708632318 after retry RPC after error@1552320902) -> (continuation of Deferred@302903604 after retry RPC after error@925662458), errback=(continuation of Deferred@404744473 after retry RPC after error@438484049) -> (continuation of Deferred@2142216031 after retry RPC after error@1529131473) -> (continuation of Deferred@1007996045 after retry RPC after error@870006634) -> (continuation of Deferred@806161193 after retry RPC after error@1322880704) -> (continuation of Deferred@1412645682 after retry RPC after error@1056507569) -> (continuation of Deferred@1718945284 after retry RPC after error@1691888516) -> (continuation of Deferred@708632318 after retry RPC after error@1552320902) -> (continuation of Deferred@302903604 after retry RPC after error@925662458)))
20:28:17.070 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:17.071 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:17.071 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:17.071 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:17.071 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:17.071 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:17.071 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:17.071 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:17.072 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:17.090 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:17.091 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:17.091 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:17.091 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:17.091 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:17.091 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:17.091 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:17.091 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:17.092 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:17.110 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:17.110 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:17.111 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:17.111 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:17.111 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:17.111 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:17.111 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:17.111 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:17.112 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:17.131 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:17.131 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:17.132 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:17.132 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:17.132 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:17.132 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:17.132 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:17.132 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:17.132 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:17.150 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:17.150 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:17.151 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:17.151 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:17.151 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:17.151 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:17.151 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:17.151 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:17.152 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:17.191 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:17.191 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:17.192 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:17.192 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:17.192 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:17.192 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:17.192 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:17.192 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:17.193 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:17.211 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:17.211 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:17.211 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:17.213 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:17.213 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:17.213 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:17.213 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:17.213 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:17.213 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:17.311 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:17.311 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:17.311 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:17.311 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:17.311 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:17.311 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:17.311 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:17.312 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:17.312 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:17.313 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [140ms] refreshing cache from master, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [143ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [143ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [143ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@488467324(state=PENDING, result=null, callback=(continuation of Deferred@1104426641 after retry RPC after error@787437190) -> (continuation of Deferred@1857079876 after retry RPC after error@402655845) -> (continuation of Deferred@2021705546 after retry RPC after error@739243450) -> (continuation of Deferred@1141237909 after retry RPC after error@1557041490) -> (continuation of Deferred@571123478 after retry RPC after error@1903899848) -> (continuation of Deferred@441531186 after retry RPC after error@851279147) -> (continuation of Deferred@1165512317 after retry RPC after error@1829629806), errback=(continuation of Deferred@1104426641 after retry RPC after error@787437190) -> (continuation of Deferred@1857079876 after retry RPC after error@402655845) -> (continuation of Deferred@2021705546 after retry RPC after error@739243450) -> (continuation of Deferred@1141237909 after retry RPC after error@1557041490) -> (continuation of Deferred@571123478 after retry RPC after error@1903899848) -> (continuation of Deferred@441531186 after retry RPC after error@851279147) -> (continuation of Deferred@1165512317 after retry RPC after error@1829629806)))
20:28:17.313 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [140ms] refreshing cache from master, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [143ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [143ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [143ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@488467324(state=PENDING, result=null, callback=(continuation of Deferred@1104426641 after retry RPC after error@787437190) -> (continuation of Deferred@1857079876 after retry RPC after error@402655845) -> (continuation of Deferred@2021705546 after retry RPC after error@739243450) -> (continuation of Deferred@1141237909 after retry RPC after error@1557041490) -> (continuation of Deferred@571123478 after retry RPC after error@1903899848) -> (continuation of Deferred@441531186 after retry RPC after error@851279147) -> (continuation of Deferred@1165512317 after retry RPC after error@1829629806), errback=(continuation of Deferred@1104426641 after retry RPC after error@787437190) -> (continuation of Deferred@1857079876 after retry RPC after error@402655845) -> (continuation of Deferred@2021705546 after retry RPC after error@739243450) -> (continuation of Deferred@1141237909 after retry RPC after error@1557041490) -> (continuation of Deferred@571123478 after retry RPC after error@1903899848) -> (continuation of Deferred@441531186 after retry RPC after error@851279147) -> (continuation of Deferred@1165512317 after retry RPC after error@1829629806)))
20:28:17.330 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:17.331 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:17.331 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:17.331 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:17.331 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:17.331 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:17.331 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:17.331 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:17.332 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:17.350 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:17.351 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:17.351 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:17.351 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:17.351 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:17.351 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:17.351 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:17.351 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:17.352 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:17.370 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:17.370 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:17.371 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:17.371 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:17.371 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:17.371 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:17.371 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:17.371 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:17.372 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:17.391 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:17.391 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:17.391 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:17.391 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:17.392 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:17.392 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:17.392 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:17.392 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:17.392 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:17.411 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:17.411 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:17.411 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:17.411 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:17.411 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:17.411 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:17.411 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:17.411 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:17.412 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:17.430 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:17.431 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:17.431 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:17.431 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:17.431 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:17.431 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:17.431 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:17.431 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:17.432 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:17.491 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:17.491 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:17.491 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:17.491 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:17.491 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:17.491 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:17.491 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:17.491 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:17.492 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:17.531 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:17.531 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:17.531 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:17.531 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:17.531 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:17.531 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:17.531 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:17.531 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:17.532 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:17.532 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=201), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, deferred=Deferred@1781989946(state=PENDING, result=null, callback=(continuation of Deferred@279578521 after retry RPC after error@135311672) -> (continuation of Deferred@898955912 after retry RPC after error@1742735570) -> (continuation of Deferred@657874296 after retry RPC after error@706916823) -> (continuation of Deferred@955276961 after retry RPC after error@1597073397) -> (continuation of Deferred@1249455350 after retry RPC after error@1969918055) -> (continuation of Deferred@1339411773 after retry RPC after error@453973501) -> (continuation of Deferred@403870966 after retry RPC after error@970643003), errback=(continuation of Deferred@279578521 after retry RPC after error@135311672) -> (continuation of Deferred@898955912 after retry RPC after error@1742735570) -> (continuation of Deferred@657874296 after retry RPC after error@706916823) -> (continuation of Deferred@955276961 after retry RPC after error@1597073397) -> (continuation of Deferred@1249455350 after retry RPC after error@1969918055) -> (continuation of Deferred@1339411773 after retry RPC after error@453973501) -> (continuation of Deferred@403870966 after retry RPC after error@970643003)))
20:28:17.533 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=201), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, deferred=Deferred@1781989946(state=PENDING, result=null, callback=(continuation of Deferred@279578521 after retry RPC after error@135311672) -> (continuation of Deferred@898955912 after retry RPC after error@1742735570) -> (continuation of Deferred@657874296 after retry RPC after error@706916823) -> (continuation of Deferred@955276961 after retry RPC after error@1597073397) -> (continuation of Deferred@1249455350 after retry RPC after error@1969918055) -> (continuation of Deferred@1339411773 after retry RPC after error@453973501) -> (continuation of Deferred@403870966 after retry RPC after error@970643003), errback=(continuation of Deferred@279578521 after retry RPC after error@135311672) -> (continuation of Deferred@898955912 after retry RPC after error@1742735570) -> (continuation of Deferred@657874296 after retry RPC after error@706916823) -> (continuation of Deferred@955276961 after retry RPC after error@1597073397) -> (continuation of Deferred@1249455350 after retry RPC after error@1969918055) -> (continuation of Deferred@1339411773 after retry RPC after error@453973501) -> (continuation of Deferred@403870966 after retry RPC after error@970643003)))
20:28:17.551 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:17.551 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:17.551 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:17.551 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:17.551 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:17.551 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:17.551 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:17.551 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:17.552 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:17.570 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:17.571 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:17.571 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:17.571 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:17.571 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:17.571 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:17.571 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:17.571 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:17.572 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:17.590 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:17.591 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:17.591 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:17.591 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:17.591 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:17.591 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:17.591 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:17.591 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:17.592 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:17.610 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:17.611 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:17.611 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:17.611 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:17.611 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:17.611 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:17.611 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:17.611 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:17.612 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:17.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:28:17.620231 3893 master_proxy_rpc.cc:177] Failed to determine new leader Master: Timed out: ConnectToClusterRpc(addrs: 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127, num_attempts: 97) passed its deadline: Not found: no leader found: ConnectToClusterRpc(addrs: 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127, num_attempts: 1)
20:28:17.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:28:17.622704 3893 meta_cache.cc:788] Timed out: LookupRpcById { tablet: 'f9ccfdb32e164e61bde68775baab7f9d', attempt: 2 } failed: LookupRpcById timed out after deadline expired: LookupRpcById { tablet: 'f9ccfdb32e164e61bde68775baab7f9d', attempt: 2 } passed its deadline: Timed out: ConnectToClusterRpc(addrs: 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127, num_attempts: 97) passed its deadline: Not found: no leader found: ConnectToClusterRpc(addrs: 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127, num_attempts: 1)
20:28:17.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:28:17.625168 3893 master_proxy_rpc.cc:203] Re-attempting LookupRpcById request to leader Master (127.3.46.62:46071)
20:28:17.630 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:17.630 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:17.631 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:17.631 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:17.631 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:17.631 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:17.631 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:17.631 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:17.632 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:17.651 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:17.651 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:17.652 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:17.652 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:17.652 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:17.652 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:17.652 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:17.652 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:17.652 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:17.710 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:17.710 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:17.711 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:17.711 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:17.711 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:17.712 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:17.712 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:17.712 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:17.712 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:17.713 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@979124199(state=PENDING, result=null, callback=(continuation of Deferred@561238634 after retry RPC after error@2056287982) -> (continuation of Deferred@1839587583 after retry RPC after error@920476075) -> (continuation of Deferred@159530392 after retry RPC after error@1885080896) -> (continuation of Deferred@842840492 after retry RPC after error@1861271105) -> (continuation of Deferred@1980113146 after retry RPC after error@1721763191) -> (continuation of Deferred@822485462 after retry RPC after error@1525650177), errback=(continuation of Deferred@561238634 after retry RPC after error@2056287982) -> (continuation of Deferred@1839587583 after retry RPC after error@920476075) -> (continuation of Deferred@159530392 after retry RPC after error@1885080896) -> (continuation of Deferred@842840492 after retry RPC after error@1861271105) -> (continuation of Deferred@1980113146 after retry RPC after error@1721763191) -> (continuation of Deferred@822485462 after retry RPC after error@1525650177)))
20:28:17.713 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@979124199(state=PENDING, result=null, callback=(continuation of Deferred@561238634 after retry RPC after error@2056287982) -> (continuation of Deferred@1839587583 after retry RPC after error@920476075) -> (continuation of Deferred@159530392 after retry RPC after error@1885080896) -> (continuation of Deferred@842840492 after retry RPC after error@1861271105) -> (continuation of Deferred@1980113146 after retry RPC after error@1721763191) -> (continuation of Deferred@822485462 after retry RPC after error@1525650177), errback=(continuation of Deferred@561238634 after retry RPC after error@2056287982) -> (continuation of Deferred@1839587583 after retry RPC after error@920476075) -> (continuation of Deferred@159530392 after retry RPC after error@1885080896) -> (continuation of Deferred@842840492 after retry RPC after error@1861271105) -> (continuation of Deferred@1980113146 after retry RPC after error@1721763191) -> (continuation of Deferred@822485462 after retry RPC after error@1525650177)))
20:28:17.731 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:17.731 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:17.732 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:17.732 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:17.732 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:17.732 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:17.732 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:17.732 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:17.733 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:17.751 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:17.751 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:17.752 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:17.752 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:17.752 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:17.752 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:17.752 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:17.752 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:17.753 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:17.771 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:17.771 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:17.771 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:17.771 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:17.771 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:17.771 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:17.771 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:17.772 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:17.772 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:17.791 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:17.791 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:17.791 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:17.791 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:17.791 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:17.791 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:17.791 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:17.791 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:17.792 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:17.810 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:17.811 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:17.811 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:17.811 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:17.811 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:17.811 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:17.811 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:17.811 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:17.812 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:17.830 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:17.831 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:17.831 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:17.831 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:17.831 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:17.831 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:17.831 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:17.832 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:17.832 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:17.891 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:17.891 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:17.891 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:17.891 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:17.891 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:17.891 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:17.891 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:17.892 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:17.892 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:17.893 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=161), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [39ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [40ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [60ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [80ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [99ms] refreshing cache from master, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [100ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [100ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [101ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [160ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [161ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@2077022726(state=PENDING, result=null, callback=(continuation of Deferred@1562196453 after retry RPC after error@195561689) -> (continuation of Deferred@1381251089 after retry RPC after error@1331025079) -> (continuation of Deferred@2075332265 after retry RPC after error@815153094) -> (continuation of Deferred@1131486581 after retry RPC after error@585140672) -> (continuation of Deferred@1095044467 after retry RPC after error@1085039250) -> (continuation of Deferred@201151815 after retry RPC after error@762671985), errback=(continuation of Deferred@1562196453 after retry RPC after error@195561689) -> (continuation of Deferred@1381251089 after retry RPC after error@1331025079) -> (continuation of Deferred@2075332265 after retry RPC after error@815153094) -> (continuation of Deferred@1131486581 after retry RPC after error@585140672) -> (continuation of Deferred@1095044467 after retry RPC after error@1085039250) -> (continuation of Deferred@201151815 after retry RPC after error@762671985)))
20:28:17.893 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=161), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [39ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [40ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [60ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [80ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [99ms] refreshing cache from master, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [100ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [100ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [101ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [160ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [161ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@2077022726(state=PENDING, result=null, callback=(continuation of Deferred@1562196453 after retry RPC after error@195561689) -> (continuation of Deferred@1381251089 after retry RPC after error@1331025079) -> (continuation of Deferred@2075332265 after retry RPC after error@815153094) -> (continuation of Deferred@1131486581 after retry RPC after error@585140672) -> (continuation of Deferred@1095044467 after retry RPC after error@1085039250) -> (continuation of Deferred@201151815 after retry RPC after error@762671985), errback=(continuation of Deferred@1562196453 after retry RPC after error@195561689) -> (continuation of Deferred@1381251089 after retry RPC after error@1331025079) -> (continuation of Deferred@2075332265 after retry RPC after error@815153094) -> (continuation of Deferred@1131486581 after retry RPC after error@585140672) -> (continuation of Deferred@1095044467 after retry RPC after error@1085039250) -> (continuation of Deferred@201151815 after retry RPC after error@762671985)))
20:28:17.910 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:17.911 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:17.911 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:17.911 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:17.911 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:17.912 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:17.912 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:17.912 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:17.912 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:17.930 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:17.930 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:17.931 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:17.931 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:17.931 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:17.931 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:17.931 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:17.931 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:17.932 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:17.950 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:17.950 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:17.951 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:17.951 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:17.951 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:17.951 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:17.951 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:17.951 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:17.952 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:17.971 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:17.971 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:17.971 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:17.972 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:17.972 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:17.972 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:17.972 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:17.972 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:17.972 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:17.990 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:17.990 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:17.991 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:17.991 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:17.991 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:17.991 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:17.991 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:17.991 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:17.992 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:18.011 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:18.011 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:18.011 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:18.011 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:18.012 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:18.012 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:18.012 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:18.012 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:18.012 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:18.051 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:18.051 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:18.051 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:18.051 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:18.051 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:18.052 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:18.052 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:18.052 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:18.052 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:18.053 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=142), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [140ms] refreshing cache from master, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@1384051307(state=PENDING, result=null, callback=(continuation of Deferred@2038618252 after retry RPC after error@44972421) -> (continuation of Deferred@1065801837 after retry RPC after error@449747036) -> (continuation of Deferred@1969714481 after retry RPC after error@108398601) -> (continuation of Deferred@255524122 after retry RPC after error@1641580761) -> (continuation of Deferred@794339118 after retry RPC after error@433636492) -> (continuation of Deferred@1104296064 after retry RPC after error@1684667939), errback=(continuation of Deferred@2038618252 after retry RPC after error@44972421) -> (continuation of Deferred@1065801837 after retry RPC after error@449747036) -> (continuation of Deferred@1969714481 after retry RPC after error@108398601) -> (continuation of Deferred@255524122 after retry RPC after error@1641580761) -> (continuation of Deferred@794339118 after retry RPC after error@433636492) -> (continuation of Deferred@1104296064 after retry RPC after error@1684667939)))
20:28:18.053 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=142), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [140ms] refreshing cache from master, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@1384051307(state=PENDING, result=null, callback=(continuation of Deferred@2038618252 after retry RPC after error@44972421) -> (continuation of Deferred@1065801837 after retry RPC after error@449747036) -> (continuation of Deferred@1969714481 after retry RPC after error@108398601) -> (continuation of Deferred@255524122 after retry RPC after error@1641580761) -> (continuation of Deferred@794339118 after retry RPC after error@433636492) -> (continuation of Deferred@1104296064 after retry RPC after error@1684667939), errback=(continuation of Deferred@2038618252 after retry RPC after error@44972421) -> (continuation of Deferred@1065801837 after retry RPC after error@449747036) -> (continuation of Deferred@1969714481 after retry RPC after error@108398601) -> (continuation of Deferred@255524122 after retry RPC after error@1641580761) -> (continuation of Deferred@794339118 after retry RPC after error@433636492) -> (continuation of Deferred@1104296064 after retry RPC after error@1684667939)))
20:28:18.071 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:18.071 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:18.071 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:18.071 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:18.071 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:18.071 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:18.071 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:18.071 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:18.072 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:18.090 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:18.091 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:18.091 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:18.091 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:18.091 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:18.091 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:18.091 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:18.091 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:18.092 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:18.110 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:18.110 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:18.111 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:18.111 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:18.111 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:18.111 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:18.111 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:18.111 [INFO - pool-40-thread-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:18.112 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:18.131 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:18.131 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:18.131 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:18.131 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:18.131 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:18.131 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:18.132 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:18.132 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:18.132 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:18.151 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:18.151 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:18.151 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:18.151 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:18.151 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:18.151 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:18.151 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:18.151 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:18.152 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:18.170 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:18.171 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:18.171 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:18.171 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:18.171 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:18.171 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:18.171 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:18.171 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:18.172 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:18.190 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:18.190 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:18.191 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:18.191 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:18.191 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:18.191 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:18.191 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:18.191 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:18.192 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:18.331 [DEBUG - pool-40-thread-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=260), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, deferred=Deferred@442527587(state=PENDING, result=null, callback=(continuation of Deferred@1886333741 after retry RPC after error@1397056294) -> (continuation of Deferred@458677624 after retry RPC after error@39105030) -> (continuation of Deferred@191612706 after retry RPC after error@1934608698) -> (continuation of Deferred@1583310646 after retry RPC after error@2033990265) -> (continuation of Deferred@742109355 after retry RPC after error@556837391) -> (continuation of Deferred@74653640 after retry RPC after error@470024909) -> (continuation of Deferred@1833649934 after retry RPC after error@1234172026), errback=(continuation of Deferred@1886333741 after retry RPC after error@1397056294) -> (continuation of Deferred@458677624 after retry RPC after error@39105030) -> (continuation of Deferred@191612706 after retry RPC after error@1934608698) -> (continuation of Deferred@1583310646 after retry RPC after error@2033990265) -> (continuation of Deferred@742109355 after retry RPC after error@556837391) -> (continuation of Deferred@74653640 after retry RPC after error@470024909) -> (continuation of Deferred@1833649934 after retry RPC after error@1234172026)))
20:28:18.332 [WARN - pool-40-thread-1] (HashedWheelTimer.java:716) An exception was thrown by TimerTask.
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713)
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701)
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:787)
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501)
at java.base/java.lang.Thread.run(Thread.java:840)
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713)
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701)
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:787)
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501)
at java.base/java.lang.Thread.run(Thread.java:840)
20:28:18.350 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:18.350 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:18.351 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:18.351 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:18.351 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:18.351 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:18.351 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:18.351 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:18.352 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:18.371 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:18.371 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:18.371 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:18.371 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:18.372 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:18.372 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:18.372 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:18.372 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:18.372 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:18.391 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:18.391 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:18.391 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:18.391 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:18.391 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:18.391 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:18.392 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:18.392 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:18.392 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:18.410 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:18.411 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:18.411 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:18.411 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:18.411 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:18.411 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:18.411 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:18.411 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:18.412 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:18.430 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:18.430 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:18.431 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:18.431 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:18.431 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:18.431 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:18.431 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:18.431 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:18.432 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:18.451 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:18.451 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:18.452 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:18.452 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:18.452 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:18.452 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:18.452 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:18.452 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:18.452 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:18.510 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:18.510 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:18.511 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:18.511 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:18.511 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:18.511 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:18.511 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:18.511 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:18.512 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:18.512 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=161), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@713420829(state=PENDING, result=null, callback=(continuation of Deferred@789821404 after retry RPC after error@636185396) -> (continuation of Deferred@883988808 after retry RPC after error@833655515) -> (continuation of Deferred@53444198 after retry RPC after error@2103646204) -> (continuation of Deferred@1294580673 after retry RPC after error@2017124085) -> (continuation of Deferred@1983450 after retry RPC after error@1705474258) -> (continuation of Deferred@1699749908 after retry RPC after error@1261668157), errback=(continuation of Deferred@789821404 after retry RPC after error@636185396) -> (continuation of Deferred@883988808 after retry RPC after error@833655515) -> (continuation of Deferred@53444198 after retry RPC after error@2103646204) -> (continuation of Deferred@1294580673 after retry RPC after error@2017124085) -> (continuation of Deferred@1983450 after retry RPC after error@1705474258) -> (continuation of Deferred@1699749908 after retry RPC after error@1261668157)))
20:28:18.512 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=161), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@713420829(state=PENDING, result=null, callback=(continuation of Deferred@789821404 after retry RPC after error@636185396) -> (continuation of Deferred@883988808 after retry RPC after error@833655515) -> (continuation of Deferred@53444198 after retry RPC after error@2103646204) -> (continuation of Deferred@1294580673 after retry RPC after error@2017124085) -> (continuation of Deferred@1983450 after retry RPC after error@1705474258) -> (continuation of Deferred@1699749908 after retry RPC after error@1261668157), errback=(continuation of Deferred@789821404 after retry RPC after error@636185396) -> (continuation of Deferred@883988808 after retry RPC after error@833655515) -> (continuation of Deferred@53444198 after retry RPC after error@2103646204) -> (continuation of Deferred@1294580673 after retry RPC after error@2017124085) -> (continuation of Deferred@1983450 after retry RPC after error@1705474258) -> (continuation of Deferred@1699749908 after retry RPC after error@1261668157)))
20:28:18.531 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:18.531 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:18.531 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:18.531 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:18.531 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:18.532 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:18.532 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:18.532 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:18.532 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:18.551 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:18.551 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:18.551 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:18.551 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:18.551 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:18.551 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:18.551 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:18.551 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:18.552 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:18.570 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:18.571 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:18.571 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:18.571 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:18.571 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:18.571 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:18.571 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:18.571 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:18.572 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:18.591 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:18.591 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:18.592 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:18.592 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:18.592 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:18.592 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:18.592 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:18.592 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:18.593 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:18.611 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:18.611 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:18.611 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:18.611 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:18.611 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:18.611 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:18.611 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:18.612 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:18.612 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:18.630 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:18.631 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:18.631 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:18.631 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:18.631 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:18.631 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:18.631 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:18.631 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:18.632 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:18.670 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:18.671 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:18.671 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:18.671 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:18.671 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:18.671 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:18.671 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:18.671 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:18.672 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:18.730 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:18.731 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:18.731 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:18.731 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:18.731 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:18.731 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:18.731 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:18.731 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:18.732 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:18.732 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=201), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [0ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [1ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [20ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [40ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [80ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [99ms] refreshing cache from master, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [100ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [100ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [101ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [139ms] refreshing cache from master, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [140ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [140ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [141ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [199ms] refreshing cache from master, [199ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [199ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [199ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [200ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [200ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [201ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@1996160383(state=PENDING, result=null, callback=(continuation of Deferred@656065996 after retry RPC after error@1253228101) -> (continuation of Deferred@1459812002 after retry RPC after error@1991436344) -> (continuation of Deferred@133654407 after retry RPC after error@1483252138) -> (continuation of Deferred@413137891 after retry RPC after error@2030904288) -> (continuation of Deferred@1174059033 after retry RPC after error@1484924157) -> (continuation of Deferred@1059425517 after retry RPC after error@791128778) -> (continuation of Deferred@1066267975 after retry RPC after error@1986108295), errback=(continuation of Deferred@656065996 after retry RPC after error@1253228101) -> (continuation of Deferred@1459812002 after retry RPC after error@1991436344) -> (continuation of Deferred@133654407 after retry RPC after error@1483252138) -> (continuation of Deferred@413137891 after retry RPC after error@2030904288) -> (continuation of Deferred@1174059033 after retry RPC after error@1484924157) -> (continuation of Deferred@1059425517 after retry RPC after error@791128778) -> (continuation of Deferred@1066267975 after retry RPC after error@1986108295)))
20:28:18.732 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=201), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [0ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [1ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [20ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [40ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [80ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [99ms] refreshing cache from master, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [100ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [100ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [101ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [139ms] refreshing cache from master, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [140ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [140ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [141ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [199ms] refreshing cache from master, [199ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [199ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [199ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [200ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [200ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [201ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@1996160383(state=PENDING, result=null, callback=(continuation of Deferred@656065996 after retry RPC after error@1253228101) -> (continuation of Deferred@1459812002 after retry RPC after error@1991436344) -> (continuation of Deferred@133654407 after retry RPC after error@1483252138) -> (continuation of Deferred@413137891 after retry RPC after error@2030904288) -> (continuation of Deferred@1174059033 after retry RPC after error@1484924157) -> (continuation of Deferred@1059425517 after retry RPC after error@791128778) -> (continuation of Deferred@1066267975 after retry RPC after error@1986108295), errback=(continuation of Deferred@656065996 after retry RPC after error@1253228101) -> (continuation of Deferred@1459812002 after retry RPC after error@1991436344) -> (continuation of Deferred@133654407 after retry RPC after error@1483252138) -> (continuation of Deferred@413137891 after retry RPC after error@2030904288) -> (continuation of Deferred@1174059033 after retry RPC after error@1484924157) -> (continuation of Deferred@1059425517 after retry RPC after error@791128778) -> (continuation of Deferred@1066267975 after retry RPC after error@1986108295)))
20:28:18.750 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:18.750 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:18.751 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:18.751 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:18.751 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:18.751 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:18.751 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:18.751 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:18.752 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:18.770 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:18.770 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:18.771 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:18.771 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:18.771 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:18.771 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:18.771 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:18.771 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:18.771 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:18.771 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:18.772 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:18.772 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:18.791 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:18.791 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:18.791 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:18.791 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:18.791 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:18.791 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:18.792 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:18.792 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:18.792 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:18.810 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:18.811 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:18.811 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:18.811 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:18.811 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:18.811 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:18.811 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:18.811 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:18.812 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:18.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:28:18.816854 4503 raft_consensus.cc:493] T 00000000000000000000000000000000 P 0db1d539946942f794b0c4ec81f8f4b4 [term 4 FOLLOWER]: Starting pre-election (detected failure of leader b1731f8308c145f686f88efd6a54b851)
20:28:18.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:28:18.817241 4503 raft_consensus.cc:515] T 00000000000000000000000000000000 P 0db1d539946942f794b0c4ec81f8f4b4 [term 4 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0db1d539946942f794b0c4ec81f8f4b4" member_type: VOTER last_known_addr { host: "127.3.46.62" port: 46071 } } peers { permanent_uuid: "d6c270b387b04f17b2a58bfcb0eb4f82" member_type: VOTER last_known_addr { host: "127.3.46.61" port: 39407 } } peers { permanent_uuid: "b1731f8308c145f686f88efd6a54b851" member_type: VOTER last_known_addr { host: "127.3.46.60" port: 36127 } }
20:28:18.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:28:18.818225 4503 leader_election.cc:290] T 00000000000000000000000000000000 P 0db1d539946942f794b0c4ec81f8f4b4 [CANDIDATE]: Term 5 pre-election: Requested pre-vote from peers d6c270b387b04f17b2a58bfcb0eb4f82 (127.3.46.61:39407), b1731f8308c145f686f88efd6a54b851 (127.3.46.60:36127)
20:28:18.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:28:18.819553 4148 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.3.46.60:36127: connect: Connection refused (error 111) [suppressed 1 similar messages]
20:28:18.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:28:18.820784 4148 leader_election.cc:336] T 00000000000000000000000000000000 P 0db1d539946942f794b0c4ec81f8f4b4 [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer b1731f8308c145f686f88efd6a54b851 (127.3.46.60:36127): Network error: Client connection negotiation failed: client connection to 127.3.46.60:36127: connect: Connection refused (error 111)
20:28:18.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:28:18.820932 4149 leader_election.cc:336] T 00000000000000000000000000000000 P 0db1d539946942f794b0c4ec81f8f4b4 [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer d6c270b387b04f17b2a58bfcb0eb4f82 (127.3.46.61:39407): Network error: Client connection negotiation failed: client connection to 127.3.46.61:39407: connect: Connection refused (error 111)
20:28:18.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:28:18.821070 4149 leader_election.cc:304] T 00000000000000000000000000000000 P 0db1d539946942f794b0c4ec81f8f4b4 [CANDIDATE]: Term 5 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 0db1d539946942f794b0c4ec81f8f4b4; no voters: b1731f8308c145f686f88efd6a54b851, d6c270b387b04f17b2a58bfcb0eb4f82
20:28:18.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:28:18.821346 4503 raft_consensus.cc:2749] T 00000000000000000000000000000000 P 0db1d539946942f794b0c4ec81f8f4b4 [term 4 FOLLOWER]: Leader pre-election lost for term 5. Reason: could not achieve majority
20:28:18.830 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:18.831 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:18.831 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:18.831 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:18.831 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:18.831 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:18.831 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:18.831 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:18.832 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:18.851 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:18.851 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:18.852 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:18.852 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:18.852 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:18.852 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:18.852 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:18.852 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:18.852 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:18.890 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:18.890 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:18.891 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:18.891 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:18.891 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:18.891 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:18.891 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:18.891 [INFO - pool-40-thread-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:18.892 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:18.951 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:18.951 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:18.952 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:18.952 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:18.952 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:18.952 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:18.952 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:18.952 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:18.954 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:18.954 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=203), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [40ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [201ms] refreshing cache from master, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [204ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [204ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@967661753(state=PENDING, result=null, callback=(continuation of Deferred@1395830476 after retry RPC after error@1444345198) -> (continuation of Deferred@291612082 after retry RPC after error@1113214903) -> (continuation of Deferred@72932862 after retry RPC after error@1703212446) -> (continuation of Deferred@2035585193 after retry RPC after error@1768116163) -> (continuation of Deferred@731306044 after retry RPC after error@1769795797) -> (continuation of Deferred@879813771 after retry RPC after error@1193191569) -> (continuation of Deferred@1523896182 after retry RPC after error@1411413665), errback=(continuation of Deferred@1395830476 after retry RPC after error@1444345198) -> (continuation of Deferred@291612082 after retry RPC after error@1113214903) -> (continuation of Deferred@72932862 after retry RPC after error@1703212446) -> (continuation of Deferred@2035585193 after retry RPC after error@1768116163) -> (continuation of Deferred@731306044 after retry RPC after error@1769795797) -> (continuation of Deferred@879813771 after retry RPC after error@1193191569) -> (continuation of Deferred@1523896182 after retry RPC after error@1411413665)))
20:28:18.954 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=203), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [40ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [201ms] refreshing cache from master, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [204ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [204ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@967661753(state=PENDING, result=null, callback=(continuation of Deferred@1395830476 after retry RPC after error@1444345198) -> (continuation of Deferred@291612082 after retry RPC after error@1113214903) -> (continuation of Deferred@72932862 after retry RPC after error@1703212446) -> (continuation of Deferred@2035585193 after retry RPC after error@1768116163) -> (continuation of Deferred@731306044 after retry RPC after error@1769795797) -> (continuation of Deferred@879813771 after retry RPC after error@1193191569) -> (continuation of Deferred@1523896182 after retry RPC after error@1411413665), errback=(continuation of Deferred@1395830476 after retry RPC after error@1444345198) -> (continuation of Deferred@291612082 after retry RPC after error@1113214903) -> (continuation of Deferred@72932862 after retry RPC after error@1703212446) -> (continuation of Deferred@2035585193 after retry RPC after error@1768116163) -> (continuation of Deferred@731306044 after retry RPC after error@1769795797) -> (continuation of Deferred@879813771 after retry RPC after error@1193191569) -> (continuation of Deferred@1523896182 after retry RPC after error@1411413665)))
20:28:18.971 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:18.971 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:18.972 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:18.972 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:18.972 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:18.972 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:18.972 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:18.972 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:18.972 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:18.991 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:18.991 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:18.991 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:18.992 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:18.992 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:18.992 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:18.992 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:18.992 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:18.992 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:19.011 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:19.011 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:19.011 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:19.011 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:19.011 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:19.011 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:19.012 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:19.012 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:19.012 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:19.031 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:19.031 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:19.031 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:19.031 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:19.031 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:19.031 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:19.031 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:19.031 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:19.032 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:19.050 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:19.051 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:19.051 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:19.051 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:19.051 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:19.051 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:19.051 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:19.051 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:19.052 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:19.070 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:19.070 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:19.071 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:19.071 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:19.071 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:19.071 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:19.071 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:19.071 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:19.073 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:19.110 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:19.111 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:19.111 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:19.111 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:19.111 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:19.111 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:19.111 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:19.111 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:19.112 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:19.211 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:19.211 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:19.211 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:19.211 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:19.211 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:19.212 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:19.212 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:19.212 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:19.212 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:19.212 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=241), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [60ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [80ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [99ms] refreshing cache from master, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [100ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [100ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [139ms] refreshing cache from master, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [140ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [140ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [141ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [239ms] refreshing cache from master, [239ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [239ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [240ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [241ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@253077783(state=PENDING, result=null, callback=(continuation of Deferred@1197867906 after retry RPC after error@1331751687) -> (continuation of Deferred@2072001575 after retry RPC after error@1509916497) -> (continuation of Deferred@779230719 after retry RPC after error@1711465445) -> (continuation of Deferred@471439878 after retry RPC after error@1052670755) -> (continuation of Deferred@1901806 after retry RPC after error@1911673539) -> (continuation of Deferred@560508753 after retry RPC after error@1824702268) -> (continuation of Deferred@68222014 after retry RPC after error@2064138677), errback=(continuation of Deferred@1197867906 after retry RPC after error@1331751687) -> (continuation of Deferred@2072001575 after retry RPC after error@1509916497) -> (continuation of Deferred@779230719 after retry RPC after error@1711465445) -> (continuation of Deferred@471439878 after retry RPC after error@1052670755) -> (continuation of Deferred@1901806 after retry RPC after error@1911673539) -> (continuation of Deferred@560508753 after retry RPC after error@1824702268) -> (continuation of Deferred@68222014 after retry RPC after error@2064138677)))
20:28:19.213 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=241), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [60ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [80ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [99ms] refreshing cache from master, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [100ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [100ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [139ms] refreshing cache from master, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [140ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [140ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [141ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [239ms] refreshing cache from master, [239ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [239ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [240ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [241ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@253077783(state=PENDING, result=null, callback=(continuation of Deferred@1197867906 after retry RPC after error@1331751687) -> (continuation of Deferred@2072001575 after retry RPC after error@1509916497) -> (continuation of Deferred@779230719 after retry RPC after error@1711465445) -> (continuation of Deferred@471439878 after retry RPC after error@1052670755) -> (continuation of Deferred@1901806 after retry RPC after error@1911673539) -> (continuation of Deferred@560508753 after retry RPC after error@1824702268) -> (continuation of Deferred@68222014 after retry RPC after error@2064138677), errback=(continuation of Deferred@1197867906 after retry RPC after error@1331751687) -> (continuation of Deferred@2072001575 after retry RPC after error@1509916497) -> (continuation of Deferred@779230719 after retry RPC after error@1711465445) -> (continuation of Deferred@471439878 after retry RPC after error@1052670755) -> (continuation of Deferred@1901806 after retry RPC after error@1911673539) -> (continuation of Deferred@560508753 after retry RPC after error@1824702268) -> (continuation of Deferred@68222014 after retry RPC after error@2064138677)))
20:28:19.230 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:19.231 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:19.231 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:19.231 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:19.231 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:19.231 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:19.231 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:19.231 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:19.232 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:19.250 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:19.251 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:19.251 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:19.251 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:19.251 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:19.251 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:19.251 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:19.251 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:19.252 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:19.270 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:19.270 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:19.271 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:19.271 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:19.271 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:19.271 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:19.272 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:19.272 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:19.272 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:19.290 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:19.290 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:19.291 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:19.291 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:19.291 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:19.291 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:19.291 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:19.291 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:19.292 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:19.310 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:19.310 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:19.311 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:19.311 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:19.311 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:19.311 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:19.311 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:19.311 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:19.312 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:19.350 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:19.350 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:19.351 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:19.351 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:19.351 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:19.351 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:19.351 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:19.351 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:19.352 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:19.390 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:19.390 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:19.391 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:19.391 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:19.391 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:19.391 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:19.391 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:19.391 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:19.391 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:19.470 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:19.470 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:19.471 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:19.471 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:19.471 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:19.471 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:19.471 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:19.471 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:19.471 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:19.472 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=241), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [161ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, deferred=Deferred@817749609(state=PENDING, result=null, callback=(continuation of Deferred@921106876 after retry RPC after error@609512094) -> (continuation of Deferred@1745029207 after retry RPC after error@915656104) -> (continuation of Deferred@926546399 after retry RPC after error@809399612) -> (continuation of Deferred@1083151068 after retry RPC after error@1759111656) -> (continuation of Deferred@274072109 after retry RPC after error@1401033358) -> (continuation of Deferred@1873240912 after retry RPC after error@1209418213) -> (continuation of Deferred@1752266144 after retry RPC after error@732175426), errback=(continuation of Deferred@921106876 after retry RPC after error@609512094) -> (continuation of Deferred@1745029207 after retry RPC after error@915656104) -> (continuation of Deferred@926546399 after retry RPC after error@809399612) -> (continuation of Deferred@1083151068 after retry RPC after error@1759111656) -> (continuation of Deferred@274072109 after retry RPC after error@1401033358) -> (continuation of Deferred@1873240912 after retry RPC after error@1209418213) -> (continuation of Deferred@1752266144 after retry RPC after error@732175426)))
20:28:19.472 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=241), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [161ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, deferred=Deferred@817749609(state=PENDING, result=null, callback=(continuation of Deferred@921106876 after retry RPC after error@609512094) -> (continuation of Deferred@1745029207 after retry RPC after error@915656104) -> (continuation of Deferred@926546399 after retry RPC after error@809399612) -> (continuation of Deferred@1083151068 after retry RPC after error@1759111656) -> (continuation of Deferred@274072109 after retry RPC after error@1401033358) -> (continuation of Deferred@1873240912 after retry RPC after error@1209418213) -> (continuation of Deferred@1752266144 after retry RPC after error@732175426), errback=(continuation of Deferred@921106876 after retry RPC after error@609512094) -> (continuation of Deferred@1745029207 after retry RPC after error@915656104) -> (continuation of Deferred@926546399 after retry RPC after error@809399612) -> (continuation of Deferred@1083151068 after retry RPC after error@1759111656) -> (continuation of Deferred@274072109 after retry RPC after error@1401033358) -> (continuation of Deferred@1873240912 after retry RPC after error@1209418213) -> (continuation of Deferred@1752266144 after retry RPC after error@732175426)))
20:28:19.491 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:19.491 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:19.492 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:19.492 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:19.492 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:19.492 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:19.492 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:19.492 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:19.493 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:19.511 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:19.511 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:19.511 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:19.511 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:19.511 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:19.511 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:19.511 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:19.511 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:19.512 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:19.530 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:19.531 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:19.531 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:19.531 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:19.531 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:19.531 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:19.531 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:19.531 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:19.532 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:19.550 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:19.550 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:19.551 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:19.551 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:19.551 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:19.551 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:19.551 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:19.551 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:19.552 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:19.571 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:19.571 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:19.572 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:19.572 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:19.572 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:19.572 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:19.572 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:19.572 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:19.573 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:19.591 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:19.591 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:19.591 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:19.591 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:19.592 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:19.592 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:19.592 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:19.592 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:19.593 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:19.631 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:19.631 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:19.631 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:19.632 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:19.632 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:19.632 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:19.632 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:19.632 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:19.632 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:19.670 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:19.671 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:19.671 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:19.671 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:19.671 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:19.671 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:19.671 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:19.671 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:19.672 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:19.672 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=181), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [20ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [40ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [60ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [179ms] refreshing cache from master, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [180ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [180ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [181ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@203753986(state=PENDING, result=null, callback=(continuation of Deferred@1725077255 after retry RPC after error@1841154278) -> (continuation of Deferred@1461543840 after retry RPC after error@1942750811) -> (continuation of Deferred@30727562 after retry RPC after error@36556718) -> (continuation of Deferred@1049738280 after retry RPC after error@771529599) -> (continuation of Deferred@182664135 after retry RPC after error@1717552522) -> (continuation of Deferred@1436304789 after retry RPC after error@485628730) -> (continuation of Deferred@257701472 after retry RPC after error@397922708), errback=(continuation of Deferred@1725077255 after retry RPC after error@1841154278) -> (continuation of Deferred@1461543840 after retry RPC after error@1942750811) -> (continuation of Deferred@30727562 after retry RPC after error@36556718) -> (continuation of Deferred@1049738280 after retry RPC after error@771529599) -> (continuation of Deferred@182664135 after retry RPC after error@1717552522) -> (continuation of Deferred@1436304789 after retry RPC after error@485628730) -> (continuation of Deferred@257701472 after retry RPC after error@397922708)))
20:28:19.673 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=181), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [20ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [40ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [60ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [179ms] refreshing cache from master, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [180ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [180ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [181ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@203753986(state=PENDING, result=null, callback=(continuation of Deferred@1725077255 after retry RPC after error@1841154278) -> (continuation of Deferred@1461543840 after retry RPC after error@1942750811) -> (continuation of Deferred@30727562 after retry RPC after error@36556718) -> (continuation of Deferred@1049738280 after retry RPC after error@771529599) -> (continuation of Deferred@182664135 after retry RPC after error@1717552522) -> (continuation of Deferred@1436304789 after retry RPC after error@485628730) -> (continuation of Deferred@257701472 after retry RPC after error@397922708), errback=(continuation of Deferred@1725077255 after retry RPC after error@1841154278) -> (continuation of Deferred@1461543840 after retry RPC after error@1942750811) -> (continuation of Deferred@30727562 after retry RPC after error@36556718) -> (continuation of Deferred@1049738280 after retry RPC after error@771529599) -> (continuation of Deferred@182664135 after retry RPC after error@1717552522) -> (continuation of Deferred@1436304789 after retry RPC after error@485628730) -> (continuation of Deferred@257701472 after retry RPC after error@397922708)))
20:28:19.690 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:19.690 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:19.691 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:19.691 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:19.691 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:19.691 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:19.691 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:19.691 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:19.692 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:19.711 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:19.711 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:19.711 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:19.712 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:19.712 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:19.712 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:19.712 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:19.712 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:19.713 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:19.731 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:19.731 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:19.731 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:19.731 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:19.731 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:19.732 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:19.732 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:19.732 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:19.732 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:19.750 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:19.751 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:19.751 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:19.751 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:19.751 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:19.751 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:19.752 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:19.752 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:19.752 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:19.770 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:19.771 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:19.771 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:19.771 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:19.771 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:19.771 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:19.771 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:19.772 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:19.772 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:19.810 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:19.811 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:19.811 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:19.811 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:19.811 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:19.811 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:19.811 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:19.811 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:19.812 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:19.871 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:19.871 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:19.871 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:19.871 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:19.871 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:19.871 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:19.873 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:19.873 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:19.873 [WARN - kudu-nio-5] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:19.874 [DEBUG - kudu-nio-5] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=183), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [183ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [183ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, deferred=Deferred@1313834796(state=PENDING, result=null, callback=(continuation of Deferred@228690464 after retry RPC after error@1808370482) -> (continuation of Deferred@2082166155 after retry RPC after error@1323293635) -> (continuation of Deferred@1328457906 after retry RPC after error@1746479742) -> (continuation of Deferred@1317572091 after retry RPC after error@2117686826) -> (continuation of Deferred@1147669095 after retry RPC after error@1694732878) -> (continuation of Deferred@1869491264 after retry RPC after error@1532149633), errback=(continuation of Deferred@228690464 after retry RPC after error@1808370482) -> (continuation of Deferred@2082166155 after retry RPC after error@1323293635) -> (continuation of Deferred@1328457906 after retry RPC after error@1746479742) -> (continuation of Deferred@1317572091 after retry RPC after error@2117686826) -> (continuation of Deferred@1147669095 after retry RPC after error@1694732878) -> (continuation of Deferred@1869491264 after retry RPC after error@1532149633)))
20:28:19.874 [DEBUG - kudu-nio-5] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=183), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [183ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [183ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, deferred=Deferred@1313834796(state=PENDING, result=null, callback=(continuation of Deferred@228690464 after retry RPC after error@1808370482) -> (continuation of Deferred@2082166155 after retry RPC after error@1323293635) -> (continuation of Deferred@1328457906 after retry RPC after error@1746479742) -> (continuation of Deferred@1317572091 after retry RPC after error@2117686826) -> (continuation of Deferred@1147669095 after retry RPC after error@1694732878) -> (continuation of Deferred@1869491264 after retry RPC after error@1532149633), errback=(continuation of Deferred@228690464 after retry RPC after error@1808370482) -> (continuation of Deferred@2082166155 after retry RPC after error@1323293635) -> (continuation of Deferred@1328457906 after retry RPC after error@1746479742) -> (continuation of Deferred@1317572091 after retry RPC after error@2117686826) -> (continuation of Deferred@1147669095 after retry RPC after error@1694732878) -> (continuation of Deferred@1869491264 after retry RPC after error@1532149633)))
20:28:19.890 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:19.891 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:19.891 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:19.891 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:19.891 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:19.891 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:19.891 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:19.891 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:19.892 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:19.910 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:19.910 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:19.911 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:19.911 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:19.911 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:19.911 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:19.911 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:19.911 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:19.911 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:19.931 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:19.931 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:19.932 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:19.932 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:19.932 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:19.932 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:19.932 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:19.932 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:19.932 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:19.951 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:19.951 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:19.951 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:19.952 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:19.952 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:19.952 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:19.952 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:19.952 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:19.952 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:19.971 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:19.971 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:19.971 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:19.971 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:19.971 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:19.971 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:19.971 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:19.971 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:19.972 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:19.990 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:19.991 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:19.991 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:19.991 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:19.991 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:19.991 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:19.991 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:19.991 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:19.992 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:20.010 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:20.010 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:20.011 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:20.011 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:20.011 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:20.011 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:20.011 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:20.011 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:20.012 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:20.031 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:20.031 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:20.031 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:20.032 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:20.032 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:20.032 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:20.032 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:20.032 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:20.033 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:20.110 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:20.110 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:20.111 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:20.111 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:20.111 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:20.112 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:20.112 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:20.112 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:20.112 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:20.112 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=221), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [141ms] refreshing cache from master, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [143ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [222ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [222ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [222ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@798669154(state=PENDING, result=null, callback=(continuation of Deferred@1319738060 after retry RPC after error@1512910159) -> (continuation of Deferred@1675886123 after retry RPC after error@432540316) -> (continuation of Deferred@464882835 after retry RPC after error@675388927) -> (continuation of Deferred@1412381818 after retry RPC after error@233459849) -> (continuation of Deferred@982287539 after retry RPC after error@512116856) -> (continuation of Deferred@870808382 after retry RPC after error@1817578451) -> (continuation of Deferred@849454070 after retry RPC after error@516487250), errback=(continuation of Deferred@1319738060 after retry RPC after error@1512910159) -> (continuation of Deferred@1675886123 after retry RPC after error@432540316) -> (continuation of Deferred@464882835 after retry RPC after error@675388927) -> (continuation of Deferred@1412381818 after retry RPC after error@233459849) -> (continuation of Deferred@982287539 after retry RPC after error@512116856) -> (continuation of Deferred@870808382 after retry RPC after error@1817578451) -> (continuation of Deferred@849454070 after retry RPC after error@516487250)))
20:28:20.112 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=221), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [141ms] refreshing cache from master, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [143ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [222ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [222ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [222ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@798669154(state=PENDING, result=null, callback=(continuation of Deferred@1319738060 after retry RPC after error@1512910159) -> (continuation of Deferred@1675886123 after retry RPC after error@432540316) -> (continuation of Deferred@464882835 after retry RPC after error@675388927) -> (continuation of Deferred@1412381818 after retry RPC after error@233459849) -> (continuation of Deferred@982287539 after retry RPC after error@512116856) -> (continuation of Deferred@870808382 after retry RPC after error@1817578451) -> (continuation of Deferred@849454070 after retry RPC after error@516487250), errback=(continuation of Deferred@1319738060 after retry RPC after error@1512910159) -> (continuation of Deferred@1675886123 after retry RPC after error@432540316) -> (continuation of Deferred@464882835 after retry RPC after error@675388927) -> (continuation of Deferred@1412381818 after retry RPC after error@233459849) -> (continuation of Deferred@982287539 after retry RPC after error@512116856) -> (continuation of Deferred@870808382 after retry RPC after error@1817578451) -> (continuation of Deferred@849454070 after retry RPC after error@516487250)))
20:28:20.131 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:20.131 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:20.131 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:20.131 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:20.131 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:20.131 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:20.131 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:20.131 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:20.132 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:20.151 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:20.151 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:20.151 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:20.151 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:20.151 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:20.151 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:20.151 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:20.151 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:20.152 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:20.170 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:20.171 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:20.171 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:20.171 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:20.171 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:20.171 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:20.171 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:20.171 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:20.172 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:20.190 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:20.190 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:20.191 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:20.191 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:20.191 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:20.191 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:20.191 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:20.191 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:20.192 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:20.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:28:20.208544 4132 master_runner.cc:186] Error getting master registration for 127.3.46.61:39407: Network error: Client connection negotiation failed: client connection to 127.3.46.61:39407: connect: Connection refused (error 111),
20:28:20.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:28:20.208810 4132 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
20:28:20.211 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:20.211 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:20.211 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:20.211 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:20.212 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:20.212 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:20.212 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:20.212 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:20.212 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:20.251 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:20.251 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:20.251 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:20.251 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:20.252 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:20.252 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:20.252 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:20.252 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:20.252 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:20.291 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:20.291 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:20.291 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:20.291 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:20.291 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:20.291 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:20.291 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:20.291 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:20.292 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:20.371 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:20.371 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:20.372 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:20.372 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:20.372 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:20.372 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:20.372 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:20.372 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:20.373 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:20.373 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [241ms] refreshing cache from master, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [243ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [243ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@1217604481(state=PENDING, result=null, callback=(continuation of Deferred@1419307757 after retry RPC after error@2047158420) -> (continuation of Deferred@384650421 after retry RPC after error@1207153354) -> (continuation of Deferred@1420818267 after retry RPC after error@121447634) -> (continuation of Deferred@2068400228 after retry RPC after error@569823269) -> (continuation of Deferred@172551791 after retry RPC after error@1690128448) -> (continuation of Deferred@1426472667 after retry RPC after error@1122413478) -> (continuation of Deferred@252710403 after retry RPC after error@27357425), errback=(continuation of Deferred@1419307757 after retry RPC after error@2047158420) -> (continuation of Deferred@384650421 after retry RPC after error@1207153354) -> (continuation of Deferred@1420818267 after retry RPC after error@121447634) -> (continuation of Deferred@2068400228 after retry RPC after error@569823269) -> (continuation of Deferred@172551791 after retry RPC after error@1690128448) -> (continuation of Deferred@1426472667 after retry RPC after error@1122413478) -> (continuation of Deferred@252710403 after retry RPC after error@27357425)))
20:28:20.373 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [241ms] refreshing cache from master, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [243ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [243ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@1217604481(state=PENDING, result=null, callback=(continuation of Deferred@1419307757 after retry RPC after error@2047158420) -> (continuation of Deferred@384650421 after retry RPC after error@1207153354) -> (continuation of Deferred@1420818267 after retry RPC after error@121447634) -> (continuation of Deferred@2068400228 after retry RPC after error@569823269) -> (continuation of Deferred@172551791 after retry RPC after error@1690128448) -> (continuation of Deferred@1426472667 after retry RPC after error@1122413478) -> (continuation of Deferred@252710403 after retry RPC after error@27357425), errback=(continuation of Deferred@1419307757 after retry RPC after error@2047158420) -> (continuation of Deferred@384650421 after retry RPC after error@1207153354) -> (continuation of Deferred@1420818267 after retry RPC after error@121447634) -> (continuation of Deferred@2068400228 after retry RPC after error@569823269) -> (continuation of Deferred@172551791 after retry RPC after error@1690128448) -> (continuation of Deferred@1426472667 after retry RPC after error@1122413478) -> (continuation of Deferred@252710403 after retry RPC after error@27357425)))
20:28:20.391 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:20.391 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:20.391 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:20.391 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:20.392 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:20.392 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:20.392 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:20.392 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:20.392 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:20.410 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:20.411 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:20.411 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:20.411 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:20.411 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:20.411 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:20.411 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:20.411 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:20.412 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:20.430 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:20.430 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:20.431 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:20.431 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:20.431 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:20.431 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:20.431 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:20.431 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:20.432 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:20.451 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:20.451 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:20.452 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:20.452 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:20.452 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:20.452 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:20.452 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:20.452 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:20.452 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:20.471 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:20.472 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:20.472 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:20.472 [INFO - pool-40-thread-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:20.472 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:20.473 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:20.473 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:20.473 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:20.473 [WARN - kudu-nio-2] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:20.491 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:20.492 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:20.492 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:20.492 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:20.492 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:20.492 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:20.492 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:20.492 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:20.492 [WARN - kudu-nio-4] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:20.531 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:20.531 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:20.531 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:20.531 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:20.531 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:20.532 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:20.532 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:20.532 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:20.532 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:20.551 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:20.551 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:20.551 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:20.551 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:20.552 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:20.552 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:20.552 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:20.552 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:20.552 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:20.552 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=161), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [1ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [20ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [40ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [99ms] refreshing cache from master, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [101ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [139ms] refreshing cache from master, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [141ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [159ms] refreshing cache from master, [159ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [159ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [161ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, deferred=Deferred@1509344571(state=PENDING, result=null, callback=(continuation of Deferred@17086132 after retry RPC after error@768554955) -> (continuation of Deferred@1831429907 after retry RPC after error@43628688) -> (continuation of Deferred@646713185 after retry RPC after error@1915294996) -> (continuation of Deferred@1419699645 after retry RPC after error@462195334) -> (continuation of Deferred@1564309905 after retry RPC after error@1728464535) -> (continuation of Deferred@1142351047 after retry RPC after error@1001492089) -> (continuation of Deferred@234260847 after retry RPC after error@1477530808), errback=(continuation of Deferred@17086132 after retry RPC after error@768554955) -> (continuation of Deferred@1831429907 after retry RPC after error@43628688) -> (continuation of Deferred@646713185 after retry RPC after error@1915294996) -> (continuation of Deferred@1419699645 after retry RPC after error@462195334) -> (continuation of Deferred@1564309905 after retry RPC after error@1728464535) -> (continuation of Deferred@1142351047 after retry RPC after error@1001492089) -> (continuation of Deferred@234260847 after retry RPC after error@1477530808)))
20:28:20.552 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=161), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [1ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [20ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [40ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [99ms] refreshing cache from master, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [101ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [139ms] refreshing cache from master, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [141ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [159ms] refreshing cache from master, [159ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [159ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [161ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, deferred=Deferred@1509344571(state=PENDING, result=null, callback=(continuation of Deferred@17086132 after retry RPC after error@768554955) -> (continuation of Deferred@1831429907 after retry RPC after error@43628688) -> (continuation of Deferred@646713185 after retry RPC after error@1915294996) -> (continuation of Deferred@1419699645 after retry RPC after error@462195334) -> (continuation of Deferred@1564309905 after retry RPC after error@1728464535) -> (continuation of Deferred@1142351047 after retry RPC after error@1001492089) -> (continuation of Deferred@234260847 after retry RPC after error@1477530808), errback=(continuation of Deferred@17086132 after retry RPC after error@768554955) -> (continuation of Deferred@1831429907 after retry RPC after error@43628688) -> (continuation of Deferred@646713185 after retry RPC after error@1915294996) -> (continuation of Deferred@1419699645 after retry RPC after error@462195334) -> (continuation of Deferred@1564309905 after retry RPC after error@1728464535) -> (continuation of Deferred@1142351047 after retry RPC after error@1001492089) -> (continuation of Deferred@234260847 after retry RPC after error@1477530808)))
20:28:20.571 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:20.571 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:20.571 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:20.571 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:20.571 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:20.571 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:20.572 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:20.572 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:20.572 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:20.591 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:20.591 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:20.592 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:20.592 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:20.592 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:20.592 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:20.592 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:20.592 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:20.592 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:20.611 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:20.611 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:20.611 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:20.611 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:20.612 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:20.611 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:20.612 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:20.612 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:20.613 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:20.630 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:20.631 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:20.631 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:20.631 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:20.631 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:20.631 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:20.631 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:20.631 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:20.632 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:20.650 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:20.650 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:20.651 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:20.651 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:20.651 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:20.651 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:20.651 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:20.651 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:20.652 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:20.671 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:20.671 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:20.672 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:20.672 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:20.672 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:20.672 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:20.672 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:20.672 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:20.672 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:20.711 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:20.711 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:20.712 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:20.712 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:20.712 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:20.712 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:20.712 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:20.712 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:20.712 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:20.751 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:20.751 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:20.752 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:20.752 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:20.752 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:20.752 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:20.752 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:20.752 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:20.752 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:20.753 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [141ms] refreshing cache from master, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [181ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, deferred=Deferred@1228186516(state=PENDING, result=null, callback=(continuation of Deferred@1822492756 after retry RPC after error@1389577267) -> (continuation of Deferred@251668854 after retry RPC after error@1665401601) -> (continuation of Deferred@121848981 after retry RPC after error@1341749906) -> (continuation of Deferred@1102885250 after retry RPC after error@44302015) -> (continuation of Deferred@525952402 after retry RPC after error@460366081) -> (continuation of Deferred@1063531340 after retry RPC after error@581919769) -> (continuation of Deferred@1980690475 after retry RPC after error@1760046914), errback=(continuation of Deferred@1822492756 after retry RPC after error@1389577267) -> (continuation of Deferred@251668854 after retry RPC after error@1665401601) -> (continuation of Deferred@121848981 after retry RPC after error@1341749906) -> (continuation of Deferred@1102885250 after retry RPC after error@44302015) -> (continuation of Deferred@525952402 after retry RPC after error@460366081) -> (continuation of Deferred@1063531340 after retry RPC after error@581919769) -> (continuation of Deferred@1980690475 after retry RPC after error@1760046914)))
20:28:20.753 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [141ms] refreshing cache from master, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [181ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, deferred=Deferred@1228186516(state=PENDING, result=null, callback=(continuation of Deferred@1822492756 after retry RPC after error@1389577267) -> (continuation of Deferred@251668854 after retry RPC after error@1665401601) -> (continuation of Deferred@121848981 after retry RPC after error@1341749906) -> (continuation of Deferred@1102885250 after retry RPC after error@44302015) -> (continuation of Deferred@525952402 after retry RPC after error@460366081) -> (continuation of Deferred@1063531340 after retry RPC after error@581919769) -> (continuation of Deferred@1980690475 after retry RPC after error@1760046914), errback=(continuation of Deferred@1822492756 after retry RPC after error@1389577267) -> (continuation of Deferred@251668854 after retry RPC after error@1665401601) -> (continuation of Deferred@121848981 after retry RPC after error@1341749906) -> (continuation of Deferred@1102885250 after retry RPC after error@44302015) -> (continuation of Deferred@525952402 after retry RPC after error@460366081) -> (continuation of Deferred@1063531340 after retry RPC after error@581919769) -> (continuation of Deferred@1980690475 after retry RPC after error@1760046914)))
20:28:20.771 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:20.771 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:20.772 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:20.772 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:20.772 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:20.772 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:20.772 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:20.772 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:20.773 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:20.791 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:20.791 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:20.791 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:20.791 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:20.791 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:20.791 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:20.791 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:20.792 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:20.792 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:20.810 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:20.811 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:20.811 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:20.811 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:20.811 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:20.811 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:20.811 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:20.811 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:20.812 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:20.830 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:20.830 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:20.831 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:20.831 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:20.831 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:20.831 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:20.831 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:20.831 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:20.832 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:20.851 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:20.851 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:20.851 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:20.851 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:20.851 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:20.851 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:20.852 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:20.852 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:20.852 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:20.871 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:20.871 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:20.871 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:20.871 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:20.871 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:20.871 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:20.871 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:20.871 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:20.872 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:20.931 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:20.931 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:20.931 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:20.931 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:20.931 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:20.931 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:20.931 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:20.932 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:20.932 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:20.971 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:20.971 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:20.971 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:20.971 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:20.971 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:20.971 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:20.971 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:20.971 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:20.972 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:20.990 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:20.991 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:20.991 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:20.991 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:20.991 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:20.991 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:20.991 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:20.991 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:20.992 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:20.992 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=221), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [19ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [20ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [40ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [60ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [79ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [80ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [99ms] refreshing cache from master, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [100ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [100ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [101ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [159ms] refreshing cache from master, [159ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [160ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [161ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [199ms] refreshing cache from master, [199ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [200ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [200ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [201ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [219ms] refreshing cache from master, [219ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [219ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [220ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [220ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [221ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, deferred=Deferred@900421081(state=PENDING, result=null, callback=(continuation of Deferred@1866341978 after retry RPC after error@1162260865) -> (continuation of Deferred@456451362 after retry RPC after error@1952345600) -> (continuation of Deferred@1804431903 after retry RPC after error@456596566) -> (continuation of Deferred@655812583 after retry RPC after error@541045611) -> (continuation of Deferred@547500281 after retry RPC after error@55856407) -> (continuation of Deferred@325958469 after retry RPC after error@1318385320) -> (continuation of Deferred@2139103867 after retry RPC after error@1579017321) -> (continuation of Deferred@550820732 after retry RPC after error@1226594424), errback=(continuation of Deferred@1866341978 after retry RPC after error@1162260865) -> (continuation of Deferred@456451362 after retry RPC after error@1952345600) -> (continuation of Deferred@1804431903 after retry RPC after error@456596566) -> (continuation of Deferred@655812583 after retry RPC after error@541045611) -> (continuation of Deferred@547500281 after retry RPC after error@55856407) -> (continuation of Deferred@325958469 after retry RPC after error@1318385320) -> (continuation of Deferred@2139103867 after retry RPC after error@1579017321) -> (continuation of Deferred@550820732 after retry RPC after error@1226594424)))
20:28:20.993 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=221), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [19ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [20ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [40ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [60ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [79ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [80ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [99ms] refreshing cache from master, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [100ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [100ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [101ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [159ms] refreshing cache from master, [159ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [160ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [161ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [199ms] refreshing cache from master, [199ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [200ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [200ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [201ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [219ms] refreshing cache from master, [219ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [219ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [220ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [220ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [221ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, deferred=Deferred@900421081(state=PENDING, result=null, callback=(continuation of Deferred@1866341978 after retry RPC after error@1162260865) -> (continuation of Deferred@456451362 after retry RPC after error@1952345600) -> (continuation of Deferred@1804431903 after retry RPC after error@456596566) -> (continuation of Deferred@655812583 after retry RPC after error@541045611) -> (continuation of Deferred@547500281 after retry RPC after error@55856407) -> (continuation of Deferred@325958469 after retry RPC after error@1318385320) -> (continuation of Deferred@2139103867 after retry RPC after error@1579017321) -> (continuation of Deferred@550820732 after retry RPC after error@1226594424), errback=(continuation of Deferred@1866341978 after retry RPC after error@1162260865) -> (continuation of Deferred@456451362 after retry RPC after error@1952345600) -> (continuation of Deferred@1804431903 after retry RPC after error@456596566) -> (continuation of Deferred@655812583 after retry RPC after error@541045611) -> (continuation of Deferred@547500281 after retry RPC after error@55856407) -> (continuation of Deferred@325958469 after retry RPC after error@1318385320) -> (continuation of Deferred@2139103867 after retry RPC after error@1579017321) -> (continuation of Deferred@550820732 after retry RPC after error@1226594424)))
20:28:21.010 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:21.011 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:21.011 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:21.011 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:21.011 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:21.011 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:21.011 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:21.011 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:21.012 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:21.030 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:21.031 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:21.031 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:21.031 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:21.031 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:21.031 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:21.031 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:21.032 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:21.032 [WARN - kudu-nio-6] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:21.050 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:21.050 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:21.051 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:21.051 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:21.051 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:21.051 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:21.051 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:21.051 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:21.052 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:21.071 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:21.071 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:21.072 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:21.072 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:21.072 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:21.072 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:21.072 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:21.072 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:21.073 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:21.091 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:21.091 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:21.091 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:21.091 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:21.091 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:21.091 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:21.092 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:21.092 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:21.092 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:21.131 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:21.131 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:21.131 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:21.131 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:21.131 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:21.131 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:21.131 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:21.132 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:21.132 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:21.171 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:21.171 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:21.171 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:21.171 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:21.171 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:21.171 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:21.172 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:21.172 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:21.172 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:21.173 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [63ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, deferred=Deferred@352982469(state=PENDING, result=null, callback=(continuation of Deferred@1717578273 after retry RPC after error@1525174980) -> (continuation of Deferred@1864548857 after retry RPC after error@717440485) -> (continuation of Deferred@1293239230 after retry RPC after error@1433618090) -> (continuation of Deferred@749168884 after retry RPC after error@486276328) -> (continuation of Deferred@2064925818 after retry RPC after error@882707364) -> (continuation of Deferred@602547467 after retry RPC after error@1614123298), errback=(continuation of Deferred@1717578273 after retry RPC after error@1525174980) -> (continuation of Deferred@1864548857 after retry RPC after error@717440485) -> (continuation of Deferred@1293239230 after retry RPC after error@1433618090) -> (continuation of Deferred@749168884 after retry RPC after error@486276328) -> (continuation of Deferred@2064925818 after retry RPC after error@882707364) -> (continuation of Deferred@602547467 after retry RPC after error@1614123298)))
20:28:21.173 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [63ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, deferred=Deferred@352982469(state=PENDING, result=null, callback=(continuation of Deferred@1717578273 after retry RPC after error@1525174980) -> (continuation of Deferred@1864548857 after retry RPC after error@717440485) -> (continuation of Deferred@1293239230 after retry RPC after error@1433618090) -> (continuation of Deferred@749168884 after retry RPC after error@486276328) -> (continuation of Deferred@2064925818 after retry RPC after error@882707364) -> (continuation of Deferred@602547467 after retry RPC after error@1614123298), errback=(continuation of Deferred@1717578273 after retry RPC after error@1525174980) -> (continuation of Deferred@1864548857 after retry RPC after error@717440485) -> (continuation of Deferred@1293239230 after retry RPC after error@1433618090) -> (continuation of Deferred@749168884 after retry RPC after error@486276328) -> (continuation of Deferred@2064925818 after retry RPC after error@882707364) -> (continuation of Deferred@602547467 after retry RPC after error@1614123298)))
20:28:21.190 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:21.191 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:21.191 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:21.191 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:21.191 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:21.191 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:21.191 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:21.191 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:21.192 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:21.211 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:21.211 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:21.212 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:21.212 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:21.212 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:21.212 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:21.212 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:21.212 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:21.213 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:21.231 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:21.231 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:21.232 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:21.232 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:21.232 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:21.232 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:21.232 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:21.232 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:21.233 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:21.251 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:21.251 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:21.251 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:21.252 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:21.252 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:21.252 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:21.252 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:21.252 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:21.252 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:21.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:28:21.258456 4523 raft_consensus.cc:493] T 00000000000000000000000000000000 P 0db1d539946942f794b0c4ec81f8f4b4 [term 4 FOLLOWER]: Starting pre-election (detected failure of leader b1731f8308c145f686f88efd6a54b851)
20:28:21.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:28:21.258677 4523 raft_consensus.cc:515] T 00000000000000000000000000000000 P 0db1d539946942f794b0c4ec81f8f4b4 [term 4 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0db1d539946942f794b0c4ec81f8f4b4" member_type: VOTER last_known_addr { host: "127.3.46.62" port: 46071 } } peers { permanent_uuid: "d6c270b387b04f17b2a58bfcb0eb4f82" member_type: VOTER last_known_addr { host: "127.3.46.61" port: 39407 } } peers { permanent_uuid: "b1731f8308c145f686f88efd6a54b851" member_type: VOTER last_known_addr { host: "127.3.46.60" port: 36127 } }
20:28:21.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:28:21.259290 4523 leader_election.cc:290] T 00000000000000000000000000000000 P 0db1d539946942f794b0c4ec81f8f4b4 [CANDIDATE]: Term 5 pre-election: Requested pre-vote from peers d6c270b387b04f17b2a58bfcb0eb4f82 (127.3.46.61:39407), b1731f8308c145f686f88efd6a54b851 (127.3.46.60:36127)
20:28:21.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:28:21.262076 4148 leader_election.cc:336] T 00000000000000000000000000000000 P 0db1d539946942f794b0c4ec81f8f4b4 [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer b1731f8308c145f686f88efd6a54b851 (127.3.46.60:36127): Network error: Client connection negotiation failed: client connection to 127.3.46.60:36127: connect: Connection refused (error 111)
20:28:21.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:28:21.262264 4149 leader_election.cc:336] T 00000000000000000000000000000000 P 0db1d539946942f794b0c4ec81f8f4b4 [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer d6c270b387b04f17b2a58bfcb0eb4f82 (127.3.46.61:39407): Network error: Client connection negotiation failed: client connection to 127.3.46.61:39407: connect: Connection refused (error 111)
20:28:21.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:28:21.262418 4149 leader_election.cc:304] T 00000000000000000000000000000000 P 0db1d539946942f794b0c4ec81f8f4b4 [CANDIDATE]: Term 5 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 0db1d539946942f794b0c4ec81f8f4b4; no voters: b1731f8308c145f686f88efd6a54b851, d6c270b387b04f17b2a58bfcb0eb4f82
20:28:21.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:28:21.262728 4523 raft_consensus.cc:2749] T 00000000000000000000000000000000 P 0db1d539946942f794b0c4ec81f8f4b4 [term 4 FOLLOWER]: Leader pre-election lost for term 5. Reason: could not achieve majority
20:28:21.271 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:21.271 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:21.271 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:21.271 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:21.272 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:21.272 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:21.272 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:21.272 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:21.272 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:21.291 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:21.291 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:21.291 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:21.291 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:21.291 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:21.291 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:21.292 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:21.292 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:21.292 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:21.310 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:21.311 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:21.311 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:21.311 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:21.311 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:21.311 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:21.311 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:21.311 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:21.312 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:21.391 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:21.391 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:21.391 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:21.391 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:21.391 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:21.391 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:21.391 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:21.391 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:21.392 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:21.392 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [23ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [43ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@994743865(state=PENDING, result=null, callback=(continuation of Deferred@1615194012 after retry RPC after error@1276375253) -> (continuation of Deferred@763506444 after retry RPC after error@1108778971) -> (continuation of Deferred@228653314 after retry RPC after error@1592145211) -> (continuation of Deferred@1654692772 after retry RPC after error@339621996) -> (continuation of Deferred@509393664 after retry RPC after error@1912347165) -> (continuation of Deferred@401064509 after retry RPC after error@1929344501) -> (continuation of Deferred@159400977 after retry RPC after error@415075278), errback=(continuation of Deferred@1615194012 after retry RPC after error@1276375253) -> (continuation of Deferred@763506444 after retry RPC after error@1108778971) -> (continuation of Deferred@228653314 after retry RPC after error@1592145211) -> (continuation of Deferred@1654692772 after retry RPC after error@339621996) -> (continuation of Deferred@509393664 after retry RPC after error@1912347165) -> (continuation of Deferred@401064509 after retry RPC after error@1929344501) -> (continuation of Deferred@159400977 after retry RPC after error@415075278)))
20:28:21.393 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [23ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [43ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@994743865(state=PENDING, result=null, callback=(continuation of Deferred@1615194012 after retry RPC after error@1276375253) -> (continuation of Deferred@763506444 after retry RPC after error@1108778971) -> (continuation of Deferred@228653314 after retry RPC after error@1592145211) -> (continuation of Deferred@1654692772 after retry RPC after error@339621996) -> (continuation of Deferred@509393664 after retry RPC after error@1912347165) -> (continuation of Deferred@401064509 after retry RPC after error@1929344501) -> (continuation of Deferred@159400977 after retry RPC after error@415075278), errback=(continuation of Deferred@1615194012 after retry RPC after error@1276375253) -> (continuation of Deferred@763506444 after retry RPC after error@1108778971) -> (continuation of Deferred@228653314 after retry RPC after error@1592145211) -> (continuation of Deferred@1654692772 after retry RPC after error@339621996) -> (continuation of Deferred@509393664 after retry RPC after error@1912347165) -> (continuation of Deferred@401064509 after retry RPC after error@1929344501) -> (continuation of Deferred@159400977 after retry RPC after error@415075278)))
20:28:21.410 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:21.411 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:21.411 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:21.411 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:21.411 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:21.411 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:21.411 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:21.411 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:21.412 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:21.430 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:21.430 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:21.431 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:21.431 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:21.431 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:21.431 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:21.431 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:21.431 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:21.432 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:21.451 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:21.451 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:21.452 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:21.452 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:21.452 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:21.452 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:21.452 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:21.452 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:21.452 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:21.471 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:21.471 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:21.472 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:21.472 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:21.472 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:21.472 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:21.472 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:21.472 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:21.473 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:21.491 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:21.491 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:21.492 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:21.492 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:21.492 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:21.492 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:21.492 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:21.492 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:21.492 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:21.511 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:21.511 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:21.511 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:21.511 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:21.511 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:21.512 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:21.512 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:21.512 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:21.512 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:21.571 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:21.571 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:21.571 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:21.571 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:21.571 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:21.571 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:21.571 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:21.572 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:21.572 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:21.573 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [63ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [100ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [160ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@856732672(state=PENDING, result=null, callback=(continuation of Deferred@1200073303 after retry RPC after error@1684034141) -> (continuation of Deferred@37946388 after retry RPC after error@876937448) -> (continuation of Deferred@1324011813 after retry RPC after error@679378975) -> (continuation of Deferred@839340704 after retry RPC after error@785991240) -> (continuation of Deferred@908253519 after retry RPC after error@1778436588) -> (continuation of Deferred@1645876318 after retry RPC after error@102102355), errback=(continuation of Deferred@1200073303 after retry RPC after error@1684034141) -> (continuation of Deferred@37946388 after retry RPC after error@876937448) -> (continuation of Deferred@1324011813 after retry RPC after error@679378975) -> (continuation of Deferred@839340704 after retry RPC after error@785991240) -> (continuation of Deferred@908253519 after retry RPC after error@1778436588) -> (continuation of Deferred@1645876318 after retry RPC after error@102102355)))
20:28:21.573 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [63ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [100ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [160ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@856732672(state=PENDING, result=null, callback=(continuation of Deferred@1200073303 after retry RPC after error@1684034141) -> (continuation of Deferred@37946388 after retry RPC after error@876937448) -> (continuation of Deferred@1324011813 after retry RPC after error@679378975) -> (continuation of Deferred@839340704 after retry RPC after error@785991240) -> (continuation of Deferred@908253519 after retry RPC after error@1778436588) -> (continuation of Deferred@1645876318 after retry RPC after error@102102355), errback=(continuation of Deferred@1200073303 after retry RPC after error@1684034141) -> (continuation of Deferred@37946388 after retry RPC after error@876937448) -> (continuation of Deferred@1324011813 after retry RPC after error@679378975) -> (continuation of Deferred@839340704 after retry RPC after error@785991240) -> (continuation of Deferred@908253519 after retry RPC after error@1778436588) -> (continuation of Deferred@1645876318 after retry RPC after error@102102355)))
20:28:21.590 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:21.591 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:21.591 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:21.591 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:21.591 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:21.591 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:21.591 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:21.591 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:21.592 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:21.610 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:21.611 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:21.611 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:21.611 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:21.611 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:21.611 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:21.611 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:21.611 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:21.612 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:21.630 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:21.630 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:21.631 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:21.631 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:21.631 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:21.631 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:21.631 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:21.631 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:21.631 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:21.651 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:21.651 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:21.652 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:21.652 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:21.652 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:21.652 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:21.652 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:21.652 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:21.652 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:21.671 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:21.671 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:21.671 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:21.671 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:21.671 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:21.671 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:21.672 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:21.672 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:21.672 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:21.691 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:21.691 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:21.691 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:21.691 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:21.691 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:21.691 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:21.691 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:21.691 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:21.692 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:21.730 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:21.731 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:21.731 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:21.731 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:21.731 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:21.731 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:21.731 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:21.731 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:21.732 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:21.732 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=141), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@572975702(state=PENDING, result=null, callback=(continuation of Deferred@2052258221 after retry RPC after error@1036407717) -> (continuation of Deferred@449272832 after retry RPC after error@254177620) -> (continuation of Deferred@1649700492 after retry RPC after error@1828733464) -> (continuation of Deferred@1292317924 after retry RPC after error@1897349313) -> (continuation of Deferred@821456993 after retry RPC after error@1602872549) -> (continuation of Deferred@727538660 after retry RPC after error@1611073286), errback=(continuation of Deferred@2052258221 after retry RPC after error@1036407717) -> (continuation of Deferred@449272832 after retry RPC after error@254177620) -> (continuation of Deferred@1649700492 after retry RPC after error@1828733464) -> (continuation of Deferred@1292317924 after retry RPC after error@1897349313) -> (continuation of Deferred@821456993 after retry RPC after error@1602872549) -> (continuation of Deferred@727538660 after retry RPC after error@1611073286)))
20:28:21.733 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=141), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@572975702(state=PENDING, result=null, callback=(continuation of Deferred@2052258221 after retry RPC after error@1036407717) -> (continuation of Deferred@449272832 after retry RPC after error@254177620) -> (continuation of Deferred@1649700492 after retry RPC after error@1828733464) -> (continuation of Deferred@1292317924 after retry RPC after error@1897349313) -> (continuation of Deferred@821456993 after retry RPC after error@1602872549) -> (continuation of Deferred@727538660 after retry RPC after error@1611073286), errback=(continuation of Deferred@2052258221 after retry RPC after error@1036407717) -> (continuation of Deferred@449272832 after retry RPC after error@254177620) -> (continuation of Deferred@1649700492 after retry RPC after error@1828733464) -> (continuation of Deferred@1292317924 after retry RPC after error@1897349313) -> (continuation of Deferred@821456993 after retry RPC after error@1602872549) -> (continuation of Deferred@727538660 after retry RPC after error@1611073286)))
20:28:21.750 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:21.751 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:21.751 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:21.751 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:21.751 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:21.751 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:21.751 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:21.751 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:21.752 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:21.770 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:21.770 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:21.771 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:21.771 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:21.771 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:21.771 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:21.771 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:21.771 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:21.771 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:21.791 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:21.791 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:21.791 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:21.791 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:21.792 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:21.792 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:21.792 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:21.792 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:21.792 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:21.811 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:21.811 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:21.811 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:21.811 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:21.812 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:21.812 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:21.812 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:21.812 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:21.812 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:21.831 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:21.831 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:21.831 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:21.831 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:21.831 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:21.831 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:21.832 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:21.832 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:21.833 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:21.871 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:21.871 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:21.871 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:21.871 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:21.871 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:21.871 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:21.872 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:21.872 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:21.872 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:21.931 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:21.931 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:21.932 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:21.932 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:21.932 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:21.932 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:21.932 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:21.932 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:21.932 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:21.991 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:21.991 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:21.992 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:21.992 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:21.992 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:21.992 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:21.992 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:21.992 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:21.993 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:21.993 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [83ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [181ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [241ms] refreshing cache from master, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [243ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [243ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, deferred=Deferred@163845674(state=PENDING, result=null, callback=(continuation of Deferred@1600362947 after retry RPC after error@1773176320) -> (continuation of Deferred@1701245671 after retry RPC after error@1287314727) -> (continuation of Deferred@265615602 after retry RPC after error@1647964046) -> (continuation of Deferred@807061732 after retry RPC after error@718501925) -> (continuation of Deferred@48764645 after retry RPC after error@796974249) -> (continuation of Deferred@1228635221 after retry RPC after error@2014852871) -> (continuation of Deferred@981817823 after retry RPC after error@323600306), errback=(continuation of Deferred@1600362947 after retry RPC after error@1773176320) -> (continuation of Deferred@1701245671 after retry RPC after error@1287314727) -> (continuation of Deferred@265615602 after retry RPC after error@1647964046) -> (continuation of Deferred@807061732 after retry RPC after error@718501925) -> (continuation of Deferred@48764645 after retry RPC after error@796974249) -> (continuation of Deferred@1228635221 after retry RPC after error@2014852871) -> (continuation of Deferred@981817823 after retry RPC after error@323600306)))
20:28:21.993 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [83ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [181ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [241ms] refreshing cache from master, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [243ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [243ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, deferred=Deferred@163845674(state=PENDING, result=null, callback=(continuation of Deferred@1600362947 after retry RPC after error@1773176320) -> (continuation of Deferred@1701245671 after retry RPC after error@1287314727) -> (continuation of Deferred@265615602 after retry RPC after error@1647964046) -> (continuation of Deferred@807061732 after retry RPC after error@718501925) -> (continuation of Deferred@48764645 after retry RPC after error@796974249) -> (continuation of Deferred@1228635221 after retry RPC after error@2014852871) -> (continuation of Deferred@981817823 after retry RPC after error@323600306), errback=(continuation of Deferred@1600362947 after retry RPC after error@1773176320) -> (continuation of Deferred@1701245671 after retry RPC after error@1287314727) -> (continuation of Deferred@265615602 after retry RPC after error@1647964046) -> (continuation of Deferred@807061732 after retry RPC after error@718501925) -> (continuation of Deferred@48764645 after retry RPC after error@796974249) -> (continuation of Deferred@1228635221 after retry RPC after error@2014852871) -> (continuation of Deferred@981817823 after retry RPC after error@323600306)))
20:28:22.011 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:22.011 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:22.012 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:22.012 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:22.012 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:22.012 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:22.012 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:22.012 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:22.013 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:22.031 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:22.031 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:22.031 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:22.031 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:22.031 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:22.031 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:22.031 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:22.031 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:22.032 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:22.050 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:22.051 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:22.051 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:22.051 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:22.051 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:22.051 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:22.051 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:22.051 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:22.052 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:22.071 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:22.071 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:22.072 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:22.072 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:22.072 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:22.072 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:22.072 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:22.072 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:22.073 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:22.091 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:22.091 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:22.091 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:22.091 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:22.091 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:22.091 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:22.092 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:22.092 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:22.092 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:22.110 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:22.111 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:22.111 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:22.111 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:22.111 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:22.112 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:22.112 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:22.112 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:22.112 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:22.150 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:22.151 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:22.151 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:22.151 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:22.151 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:22.151 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:22.151 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:22.151 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:22.152 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:22.170 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:22.171 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:22.171 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:22.171 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:22.171 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:22.171 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:22.171 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:22.171 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:22.172 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:22.172 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=161), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [20ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [40ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [79ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [80ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [99ms] refreshing cache from master, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [100ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [101ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [139ms] refreshing cache from master, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [140ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [140ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [159ms] refreshing cache from master, [159ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [159ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [160ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [160ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [161ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@1648267893(state=PENDING, result=null, callback=(continuation of Deferred@46031327 after retry RPC after error@767638301) -> (continuation of Deferred@136989715 after retry RPC after error@504482393) -> (continuation of Deferred@1875960029 after retry RPC after error@1273833449) -> (continuation of Deferred@293449811 after retry RPC after error@28238631) -> (continuation of Deferred@1771060679 after retry RPC after error@1264622963) -> (continuation of Deferred@937644900 after retry RPC after error@1092757860) -> (continuation of Deferred@691292802 after retry RPC after error@1718669778), errback=(continuation of Deferred@46031327 after retry RPC after error@767638301) -> (continuation of Deferred@136989715 after retry RPC after error@504482393) -> (continuation of Deferred@1875960029 after retry RPC after error@1273833449) -> (continuation of Deferred@293449811 after retry RPC after error@28238631) -> (continuation of Deferred@1771060679 after retry RPC after error@1264622963) -> (continuation of Deferred@937644900 after retry RPC after error@1092757860) -> (continuation of Deferred@691292802 after retry RPC after error@1718669778)))
20:28:22.172 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=161), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [20ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [40ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [79ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [80ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [99ms] refreshing cache from master, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [100ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [101ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [139ms] refreshing cache from master, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [140ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [140ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [159ms] refreshing cache from master, [159ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [159ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [160ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [160ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [161ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@1648267893(state=PENDING, result=null, callback=(continuation of Deferred@46031327 after retry RPC after error@767638301) -> (continuation of Deferred@136989715 after retry RPC after error@504482393) -> (continuation of Deferred@1875960029 after retry RPC after error@1273833449) -> (continuation of Deferred@293449811 after retry RPC after error@28238631) -> (continuation of Deferred@1771060679 after retry RPC after error@1264622963) -> (continuation of Deferred@937644900 after retry RPC after error@1092757860) -> (continuation of Deferred@691292802 after retry RPC after error@1718669778), errback=(continuation of Deferred@46031327 after retry RPC after error@767638301) -> (continuation of Deferred@136989715 after retry RPC after error@504482393) -> (continuation of Deferred@1875960029 after retry RPC after error@1273833449) -> (continuation of Deferred@293449811 after retry RPC after error@28238631) -> (continuation of Deferred@1771060679 after retry RPC after error@1264622963) -> (continuation of Deferred@937644900 after retry RPC after error@1092757860) -> (continuation of Deferred@691292802 after retry RPC after error@1718669778)))
20:28:22.190 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:22.190 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:22.191 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:22.191 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:22.191 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:22.191 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:22.191 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:22.191 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:22.192 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:22.211 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:22.211 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:22.211 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:22.212 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:22.212 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:22.212 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:22.213 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:22.213 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:22.213 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:22.230 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:22.231 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:22.231 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:22.231 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:22.231 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:22.231 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:22.231 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:22.231 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:22.232 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:22.250 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:22.251 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:22.251 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:22.251 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:22.251 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:22.251 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:22.251 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:22.251 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:22.252 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:22.270 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:22.270 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:22.271 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:22.271 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:22.271 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:22.271 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:22.271 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:22.271 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:22.272 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:22.311 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:22.311 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:22.312 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:22.312 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:22.312 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:22.312 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:22.312 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:22.312 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:22.313 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:22.370 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:22.370 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:22.371 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:22.371 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:22.371 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:22.371 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:22.371 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:22.371 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:22.372 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:22.430 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:22.430 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:22.431 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:22.431 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:22.431 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:22.431 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:22.431 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:22.431 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:22.432 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:22.432 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=241), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [23ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [23ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@362393276(state=PENDING, result=null, callback=(continuation of Deferred@1795421986 after retry RPC after error@1198388789) -> (continuation of Deferred@2128889191 after retry RPC after error@651229257) -> (continuation of Deferred@1386127702 after retry RPC after error@2053781870) -> (continuation of Deferred@645502009 after retry RPC after error@130691599) -> (continuation of Deferred@544843697 after retry RPC after error@192109465) -> (continuation of Deferred@1631706770 after retry RPC after error@936235775) -> (continuation of Deferred@1524940696 after retry RPC after error@1272613070), errback=(continuation of Deferred@1795421986 after retry RPC after error@1198388789) -> (continuation of Deferred@2128889191 after retry RPC after error@651229257) -> (continuation of Deferred@1386127702 after retry RPC after error@2053781870) -> (continuation of Deferred@645502009 after retry RPC after error@130691599) -> (continuation of Deferred@544843697 after retry RPC after error@192109465) -> (continuation of Deferred@1631706770 after retry RPC after error@936235775) -> (continuation of Deferred@1524940696 after retry RPC after error@1272613070)))
20:28:22.432 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=241), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [23ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [23ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@362393276(state=PENDING, result=null, callback=(continuation of Deferred@1795421986 after retry RPC after error@1198388789) -> (continuation of Deferred@2128889191 after retry RPC after error@651229257) -> (continuation of Deferred@1386127702 after retry RPC after error@2053781870) -> (continuation of Deferred@645502009 after retry RPC after error@130691599) -> (continuation of Deferred@544843697 after retry RPC after error@192109465) -> (continuation of Deferred@1631706770 after retry RPC after error@936235775) -> (continuation of Deferred@1524940696 after retry RPC after error@1272613070), errback=(continuation of Deferred@1795421986 after retry RPC after error@1198388789) -> (continuation of Deferred@2128889191 after retry RPC after error@651229257) -> (continuation of Deferred@1386127702 after retry RPC after error@2053781870) -> (continuation of Deferred@645502009 after retry RPC after error@130691599) -> (continuation of Deferred@544843697 after retry RPC after error@192109465) -> (continuation of Deferred@1631706770 after retry RPC after error@936235775) -> (continuation of Deferred@1524940696 after retry RPC after error@1272613070)))
20:28:22.451 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:22.451 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:22.452 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:22.452 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:22.452 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:22.452 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:22.452 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:22.452 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:22.452 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:22.471 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:22.471 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:22.471 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:22.471 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:22.471 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:22.471 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:22.471 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:22.471 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:22.472 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:22.490 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:22.491 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:22.491 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:22.491 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:22.491 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:22.491 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:22.491 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:22.491 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:22.492 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:22.510 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:22.510 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:22.511 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:22.511 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:22.511 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:22.511 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:22.511 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:22.511 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:22.512 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:22.531 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:22.531 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:22.532 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:22.532 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:22.532 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:22.532 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:22.532 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:22.532 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:22.532 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:22.551 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:22.551 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:22.552 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:22.552 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:22.552 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:22.552 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:22.552 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:22.552 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:22.552 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:22.591 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:22.591 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:22.591 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:22.591 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:22.591 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:22.591 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:22.591 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:22.592 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:22.592 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:22.630 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:22.631 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:22.631 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:22.631 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:22.631 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:22.631 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:22.631 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:22.631 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:22.632 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:22.671 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:22.671 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:22.671 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:22.671 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:22.671 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:22.671 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:22.671 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:22.672 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:22.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:28:22.671991 3893 master_proxy_rpc.cc:177] Failed to determine new leader Master: Timed out: ConnectToClusterRpc(addrs: 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127, num_attempts: 97) passed its deadline: Not found: no leader found: ConnectToClusterRpc(addrs: 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127, num_attempts: 1)
20:28:22.672 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:22.672 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=221), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [1ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [20ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [40ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [60ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [99ms] refreshing cache from master, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [101ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [139ms] refreshing cache from master, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [140ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [140ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [141ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [179ms] refreshing cache from master, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [180ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [180ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [181ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [219ms] refreshing cache from master, [219ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [219ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [220ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [220ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [221ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@2129795539(state=PENDING, result=null, callback=(continuation of Deferred@1528855430 after retry RPC after error@2115089476) -> (continuation of Deferred@112339755 after retry RPC after error@1751156399) -> (continuation of Deferred@818166894 after retry RPC after error@1124343206) -> (continuation of Deferred@1425575326 after retry RPC after error@952183888) -> (continuation of Deferred@704069738 after retry RPC after error@1249739231) -> (continuation of Deferred@1517904243 after retry RPC after error@1810593216) -> (continuation of Deferred@2022476999 after retry RPC after error@1582926192) -> (continuation of Deferred@1329207601 after retry RPC after error@2144284771), errback=(continuation of Deferred@1528855430 after retry RPC after error@2115089476) -> (continuation of Deferred@112339755 after retry RPC after error@1751156399) -> (continuation of Deferred@818166894 after retry RPC after error@1124343206) -> (continuation of Deferred@1425575326 after retry RPC after error@952183888) -> (continuation of Deferred@704069738 after retry RPC after error@1249739231) -> (continuation of Deferred@1517904243 after retry RPC after error@1810593216) -> (continuation of Deferred@2022476999 after retry RPC after error@1582926192) -> (continuation of Deferred@1329207601 after retry RPC after error@2144284771)))
20:28:22.673 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=221), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [1ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [20ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [40ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [60ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [99ms] refreshing cache from master, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [101ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [139ms] refreshing cache from master, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [140ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [140ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [141ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [179ms] refreshing cache from master, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [180ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [180ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [181ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [219ms] refreshing cache from master, [219ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [219ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [220ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [220ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [221ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@2129795539(state=PENDING, result=null, callback=(continuation of Deferred@1528855430 after retry RPC after error@2115089476) -> (continuation of Deferred@112339755 after retry RPC after error@1751156399) -> (continuation of Deferred@818166894 after retry RPC after error@1124343206) -> (continuation of Deferred@1425575326 after retry RPC after error@952183888) -> (continuation of Deferred@704069738 after retry RPC after error@1249739231) -> (continuation of Deferred@1517904243 after retry RPC after error@1810593216) -> (continuation of Deferred@2022476999 after retry RPC after error@1582926192) -> (continuation of Deferred@1329207601 after retry RPC after error@2144284771), errback=(continuation of Deferred@1528855430 after retry RPC after error@2115089476) -> (continuation of Deferred@112339755 after retry RPC after error@1751156399) -> (continuation of Deferred@818166894 after retry RPC after error@1124343206) -> (continuation of Deferred@1425575326 after retry RPC after error@952183888) -> (continuation of Deferred@704069738 after retry RPC after error@1249739231) -> (continuation of Deferred@1517904243 after retry RPC after error@1810593216) -> (continuation of Deferred@2022476999 after retry RPC after error@1582926192) -> (continuation of Deferred@1329207601 after retry RPC after error@2144284771)))
20:28:22.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:28:22.675431 3893 meta_cache.cc:788] Timed out: LookupRpcById { tablet: 'f9ccfdb32e164e61bde68775baab7f9d', attempt: 2 } failed: LookupRpcById timed out after deadline expired: LookupRpcById { tablet: 'f9ccfdb32e164e61bde68775baab7f9d', attempt: 2 } passed its deadline: Timed out: ConnectToClusterRpc(addrs: 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127, num_attempts: 97) passed its deadline: Not found: no leader found: ConnectToClusterRpc(addrs: 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127, num_attempts: 1)
20:28:22.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:28:22.677569 3893 master_proxy_rpc.cc:203] Re-attempting LookupRpcById request to leader Master (127.3.46.62:46071)
20:28:22.690 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:22.691 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:22.691 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:22.691 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:22.691 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:22.691 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:22.691 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:22.691 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:22.692 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:22.710 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:22.710 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:22.711 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:22.711 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:22.711 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:22.711 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:22.711 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:22.711 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:22.712 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:22.731 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:22.731 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:22.732 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:22.732 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:22.732 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:22.732 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:22.732 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:22.732 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:22.733 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:22.751 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:22.751 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:22.751 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:22.751 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:22.751 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:22.752 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:22.752 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:22.752 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:22.752 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:22.771 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:22.771 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:22.771 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:22.771 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:22.772 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:22.772 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:22.772 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:22.772 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:22.772 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:22.791 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:22.791 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:22.792 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:22.792 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:22.792 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:22.792 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:22.792 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:22.792 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:22.793 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:22.851 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:22.851 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:22.852 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:22.852 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:22.852 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:22.852 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:22.852 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:22.852 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:22.852 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:22.950 [DEBUG - pool-40-thread-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=259), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [43ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [103ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [161ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, deferred=Deferred@408898661(state=PENDING, result=null, callback=(continuation of Deferred@1949038835 after retry RPC after error@1881709283) -> (continuation of Deferred@734039084 after retry RPC after error@1235140949) -> (continuation of Deferred@1658410229 after retry RPC after error@1452418511) -> (continuation of Deferred@2040296157 after retry RPC after error@1178042578) -> (continuation of Deferred@1860241949 after retry RPC after error@689632851) -> (continuation of Deferred@864711479 after retry RPC after error@1229018244) -> (continuation of Deferred@789978707 after retry RPC after error@1371594124), errback=(continuation of Deferred@1949038835 after retry RPC after error@1881709283) -> (continuation of Deferred@734039084 after retry RPC after error@1235140949) -> (continuation of Deferred@1658410229 after retry RPC after error@1452418511) -> (continuation of Deferred@2040296157 after retry RPC after error@1178042578) -> (continuation of Deferred@1860241949 after retry RPC after error@689632851) -> (continuation of Deferred@864711479 after retry RPC after error@1229018244) -> (continuation of Deferred@789978707 after retry RPC after error@1371594124)))
20:28:22.951 [WARN - pool-40-thread-1] (HashedWheelTimer.java:716) An exception was thrown by TimerTask.
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713)
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701)
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:787)
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501)
at java.base/java.lang.Thread.run(Thread.java:840)
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713)
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701)
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:787)
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501)
at java.base/java.lang.Thread.run(Thread.java:840)
20:28:22.971 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:22.971 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:22.971 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:22.971 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:22.971 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:22.971 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:22.971 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:22.971 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:22.972 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:22.991 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:22.991 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:22.991 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:22.991 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:22.991 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:22.991 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:22.991 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:22.991 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:22.992 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:23.010 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:23.011 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:23.011 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:23.011 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:23.011 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:23.011 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:23.011 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:23.011 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:23.012 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:23.030 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:23.030 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:23.031 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:23.031 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:23.031 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:23.031 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:23.031 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:23.031 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:23.032 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:23.051 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:23.051 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:23.052 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:23.052 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:23.052 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:23.052 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:23.052 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:23.052 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:23.052 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:23.090 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:23.090 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:23.091 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:23.091 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:23.091 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:23.091 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:23.091 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:23.091 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:23.092 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:23.131 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:23.131 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:23.132 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:23.132 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:23.132 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:23.132 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:23.132 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:23.132 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:23.133 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:23.133 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [161ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [163ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@1688550028(state=PENDING, result=null, callback=(continuation of Deferred@931904114 after retry RPC after error@641371830) -> (continuation of Deferred@1064306444 after retry RPC after error@1925983028) -> (continuation of Deferred@419317557 after retry RPC after error@1782785442) -> (continuation of Deferred@1437861761 after retry RPC after error@1186071432) -> (continuation of Deferred@740238212 after retry RPC after error@1593998736) -> (continuation of Deferred@721416702 after retry RPC after error@2089228650), errback=(continuation of Deferred@931904114 after retry RPC after error@641371830) -> (continuation of Deferred@1064306444 after retry RPC after error@1925983028) -> (continuation of Deferred@419317557 after retry RPC after error@1782785442) -> (continuation of Deferred@1437861761 after retry RPC after error@1186071432) -> (continuation of Deferred@740238212 after retry RPC after error@1593998736) -> (continuation of Deferred@721416702 after retry RPC after error@2089228650)))
20:28:23.133 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [161ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [163ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@1688550028(state=PENDING, result=null, callback=(continuation of Deferred@931904114 after retry RPC after error@641371830) -> (continuation of Deferred@1064306444 after retry RPC after error@1925983028) -> (continuation of Deferred@419317557 after retry RPC after error@1782785442) -> (continuation of Deferred@1437861761 after retry RPC after error@1186071432) -> (continuation of Deferred@740238212 after retry RPC after error@1593998736) -> (continuation of Deferred@721416702 after retry RPC after error@2089228650), errback=(continuation of Deferred@931904114 after retry RPC after error@641371830) -> (continuation of Deferred@1064306444 after retry RPC after error@1925983028) -> (continuation of Deferred@419317557 after retry RPC after error@1782785442) -> (continuation of Deferred@1437861761 after retry RPC after error@1186071432) -> (continuation of Deferred@740238212 after retry RPC after error@1593998736) -> (continuation of Deferred@721416702 after retry RPC after error@2089228650)))
20:28:23.151 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:23.151 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:23.151 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:23.152 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:23.152 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:23.152 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:23.152 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:23.152 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:23.152 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:23.171 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:23.171 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:23.171 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:23.171 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:23.171 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:23.171 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:23.171 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:23.171 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:23.172 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:23.190 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:23.191 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:23.191 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:23.191 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:23.191 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:23.191 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:23.191 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:23.191 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:23.192 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:23.210 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:23.210 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:23.211 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:23.211 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:23.211 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:23.211 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:23.211 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:23.211 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:23.212 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:23.231 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:23.231 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:23.232 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:23.232 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:23.232 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:23.232 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:23.232 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:23.232 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:23.233 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:23.251 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:23.251 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:23.251 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:23.251 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:23.251 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:23.251 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:23.251 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:23.251 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:23.252 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:23.270 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:23.271 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:23.271 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:23.271 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:23.271 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:23.272 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:23.272 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:23.272 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:23.272 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:23.331 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:23.331 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:23.331 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:23.331 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:23.331 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:23.331 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:23.331 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:23.331 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:23.332 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:23.370 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:23.371 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:23.371 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:23.371 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:23.371 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:23.371 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:23.371 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:23.371 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:23.372 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:23.372 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=221), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [1ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [20ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [40ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [60ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [99ms] refreshing cache from master, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [100ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [100ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [101ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [119ms] refreshing cache from master, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [120ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [121ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [179ms] refreshing cache from master, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [180ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [180ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [181ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [219ms] refreshing cache from master, [219ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [219ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [220ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [220ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [221ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@680678403(state=PENDING, result=null, callback=(continuation of Deferred@732748255 after retry RPC after error@738663958) -> (continuation of Deferred@381868256 after retry RPC after error@743093367) -> (continuation of Deferred@1634071040 after retry RPC after error@1963370388) -> (continuation of Deferred@1971545100 after retry RPC after error@202608583) -> (continuation of Deferred@1118341559 after retry RPC after error@38976625) -> (continuation of Deferred@1822362419 after retry RPC after error@2086849744) -> (continuation of Deferred@2055280602 after retry RPC after error@1831312847) -> (continuation of Deferred@2046150091 after retry RPC after error@328958569), errback=(continuation of Deferred@732748255 after retry RPC after error@738663958) -> (continuation of Deferred@381868256 after retry RPC after error@743093367) -> (continuation of Deferred@1634071040 after retry RPC after error@1963370388) -> (continuation of Deferred@1971545100 after retry RPC after error@202608583) -> (continuation of Deferred@1118341559 after retry RPC after error@38976625) -> (continuation of Deferred@1822362419 after retry RPC after error@2086849744) -> (continuation of Deferred@2055280602 after retry RPC after error@1831312847) -> (continuation of Deferred@2046150091 after retry RPC after error@328958569)))
20:28:23.373 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=221), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [1ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [20ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [40ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [60ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [99ms] refreshing cache from master, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [100ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [100ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [101ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [119ms] refreshing cache from master, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [120ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [121ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [179ms] refreshing cache from master, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [180ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [180ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [181ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [219ms] refreshing cache from master, [219ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [219ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [220ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [220ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [221ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@680678403(state=PENDING, result=null, callback=(continuation of Deferred@732748255 after retry RPC after error@738663958) -> (continuation of Deferred@381868256 after retry RPC after error@743093367) -> (continuation of Deferred@1634071040 after retry RPC after error@1963370388) -> (continuation of Deferred@1971545100 after retry RPC after error@202608583) -> (continuation of Deferred@1118341559 after retry RPC after error@38976625) -> (continuation of Deferred@1822362419 after retry RPC after error@2086849744) -> (continuation of Deferred@2055280602 after retry RPC after error@1831312847) -> (continuation of Deferred@2046150091 after retry RPC after error@328958569), errback=(continuation of Deferred@732748255 after retry RPC after error@738663958) -> (continuation of Deferred@381868256 after retry RPC after error@743093367) -> (continuation of Deferred@1634071040 after retry RPC after error@1963370388) -> (continuation of Deferred@1971545100 after retry RPC after error@202608583) -> (continuation of Deferred@1118341559 after retry RPC after error@38976625) -> (continuation of Deferred@1822362419 after retry RPC after error@2086849744) -> (continuation of Deferred@2055280602 after retry RPC after error@1831312847) -> (continuation of Deferred@2046150091 after retry RPC after error@328958569)))
20:28:23.390 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:23.391 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:23.391 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:23.391 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:23.391 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:23.391 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:23.391 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:23.391 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:23.392 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:23.410 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:23.410 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:23.411 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:23.411 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:23.411 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:23.411 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:23.411 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:23.411 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:23.411 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:23.431 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:23.431 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:23.431 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:23.431 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:23.431 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:23.431 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:23.431 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:23.431 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:23.432 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:23.451 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:23.451 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:23.451 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:23.451 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:23.451 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:23.451 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:23.451 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:23.451 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:23.453 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:23.470 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:23.471 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:23.471 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:23.471 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:23.471 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:23.471 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:23.471 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:23.471 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:23.472 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:23.511 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:23.511 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:23.511 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:23.511 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:23.511 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:23.511 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:23.511 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:23.511 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:23.512 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:23.530 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:23.531 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:23.531 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:23.531 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:23.531 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:23.531 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:23.531 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:23.531 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:23.532 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:23.590 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:23.591 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:23.591 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:23.591 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:23.591 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:23.591 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:23.591 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:23.591 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:23.592 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:23.610 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:23.610 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:23.611 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:23.611 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:23.611 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:23.611 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:23.611 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:23.611 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:23.612 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:23.612 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=221), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [63ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [222ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@2059474244(state=PENDING, result=null, callback=(continuation of Deferred@940401899 after retry RPC after error@1333439482) -> (continuation of Deferred@1296943432 after retry RPC after error@1070898832) -> (continuation of Deferred@1814929341 after retry RPC after error@1989332153) -> (continuation of Deferred@1438014634 after retry RPC after error@1317894290) -> (continuation of Deferred@1314768521 after retry RPC after error@785001727) -> (continuation of Deferred@1241409152 after retry RPC after error@1813176005) -> (continuation of Deferred@1281045568 after retry RPC after error@856245472) -> (continuation of Deferred@167378535 after retry RPC after error@1944583531), errback=(continuation of Deferred@940401899 after retry RPC after error@1333439482) -> (continuation of Deferred@1296943432 after retry RPC after error@1070898832) -> (continuation of Deferred@1814929341 after retry RPC after error@1989332153) -> (continuation of Deferred@1438014634 after retry RPC after error@1317894290) -> (continuation of Deferred@1314768521 after retry RPC after error@785001727) -> (continuation of Deferred@1241409152 after retry RPC after error@1813176005) -> (continuation of Deferred@1281045568 after retry RPC after error@856245472) -> (continuation of Deferred@167378535 after retry RPC after error@1944583531)))
20:28:23.612 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=221), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [63ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [222ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@2059474244(state=PENDING, result=null, callback=(continuation of Deferred@940401899 after retry RPC after error@1333439482) -> (continuation of Deferred@1296943432 after retry RPC after error@1070898832) -> (continuation of Deferred@1814929341 after retry RPC after error@1989332153) -> (continuation of Deferred@1438014634 after retry RPC after error@1317894290) -> (continuation of Deferred@1314768521 after retry RPC after error@785001727) -> (continuation of Deferred@1241409152 after retry RPC after error@1813176005) -> (continuation of Deferred@1281045568 after retry RPC after error@856245472) -> (continuation of Deferred@167378535 after retry RPC after error@1944583531), errback=(continuation of Deferred@940401899 after retry RPC after error@1333439482) -> (continuation of Deferred@1296943432 after retry RPC after error@1070898832) -> (continuation of Deferred@1814929341 after retry RPC after error@1989332153) -> (continuation of Deferred@1438014634 after retry RPC after error@1317894290) -> (continuation of Deferred@1314768521 after retry RPC after error@785001727) -> (continuation of Deferred@1241409152 after retry RPC after error@1813176005) -> (continuation of Deferred@1281045568 after retry RPC after error@856245472) -> (continuation of Deferred@167378535 after retry RPC after error@1944583531)))
20:28:23.631 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:23.631 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:23.632 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:23.632 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:23.632 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:23.632 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:23.632 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:23.632 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:23.632 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:23.651 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:23.651 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:23.651 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:23.651 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:23.652 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:23.652 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:23.652 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:23.652 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:23.652 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:23.671 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:23.671 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:23.671 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:23.671 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:23.671 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:23.671 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:23.671 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:23.671 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:23.672 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:23.690 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:23.691 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:23.691 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:23.691 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:23.691 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:23.691 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:23.691 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:23.691 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:23.692 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:23.711 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:23.711 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:23.712 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:23.712 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:23.712 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:23.712 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:23.712 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:23.712 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:23.713 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:23.731 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:23.731 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:23.731 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:23.731 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:23.731 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:23.731 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:23.732 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:23.732 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:23.732 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:23.791 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:23.791 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:23.791 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:23.791 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:23.791 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:23.791 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:23.792 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:23.792 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:23.792 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:23.811 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:23.811 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:23.811 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:23.811 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:23.811 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:23.811 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:23.811 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:23.811 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:23.812 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:23.812 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=181), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [40ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [60ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [100ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [101ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [159ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [160ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [161ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [179ms] refreshing cache from master, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [180ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [180ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [181ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, deferred=Deferred@714570079(state=PENDING, result=null, callback=(continuation of Deferred@1467386083 after retry RPC after error@176279350) -> (continuation of Deferred@510133875 after retry RPC after error@254073184) -> (continuation of Deferred@1828092407 after retry RPC after error@1753932576) -> (continuation of Deferred@915352540 after retry RPC after error@524884916) -> (continuation of Deferred@1076879467 after retry RPC after error@2139330733) -> (continuation of Deferred@807243082 after retry RPC after error@1742644294) -> (continuation of Deferred@1489845358 after retry RPC after error@1005093162), errback=(continuation of Deferred@1467386083 after retry RPC after error@176279350) -> (continuation of Deferred@510133875 after retry RPC after error@254073184) -> (continuation of Deferred@1828092407 after retry RPC after error@1753932576) -> (continuation of Deferred@915352540 after retry RPC after error@524884916) -> (continuation of Deferred@1076879467 after retry RPC after error@2139330733) -> (continuation of Deferred@807243082 after retry RPC after error@1742644294) -> (continuation of Deferred@1489845358 after retry RPC after error@1005093162)))
20:28:23.813 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=181), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [40ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [60ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [100ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [101ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [159ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [160ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [161ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [179ms] refreshing cache from master, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [180ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [180ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [181ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, deferred=Deferred@714570079(state=PENDING, result=null, callback=(continuation of Deferred@1467386083 after retry RPC after error@176279350) -> (continuation of Deferred@510133875 after retry RPC after error@254073184) -> (continuation of Deferred@1828092407 after retry RPC after error@1753932576) -> (continuation of Deferred@915352540 after retry RPC after error@524884916) -> (continuation of Deferred@1076879467 after retry RPC after error@2139330733) -> (continuation of Deferred@807243082 after retry RPC after error@1742644294) -> (continuation of Deferred@1489845358 after retry RPC after error@1005093162), errback=(continuation of Deferred@1467386083 after retry RPC after error@176279350) -> (continuation of Deferred@510133875 after retry RPC after error@254073184) -> (continuation of Deferred@1828092407 after retry RPC after error@1753932576) -> (continuation of Deferred@915352540 after retry RPC after error@524884916) -> (continuation of Deferred@1076879467 after retry RPC after error@2139330733) -> (continuation of Deferred@807243082 after retry RPC after error@1742644294) -> (continuation of Deferred@1489845358 after retry RPC after error@1005093162)))
20:28:23.830 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:23.831 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:23.831 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:23.831 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:23.831 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:23.831 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:23.831 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:23.831 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:23.832 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:23.850 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:23.850 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:23.851 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:23.851 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:23.851 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:23.851 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:23.851 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:23.851 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:23.852 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:23.871 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:23.871 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:23.871 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:23.871 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:23.872 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:23.872 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:23.872 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:23.872 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:23.872 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:23.891 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:23.891 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:23.891 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:23.891 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:23.891 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:23.891 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:23.891 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:23.891 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:23.892 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:23.910 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:23.911 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:23.911 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:23.911 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:23.911 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:23.911 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:23.911 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:23.911 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:23.912 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:23.950 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:23.950 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:23.951 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:23.951 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:23.951 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:23.951 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:23.951 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:23.951 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:23.952 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:23.971 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:23.971 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:23.971 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:23.972 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:23.972 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:23.972 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:23.972 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:23.972 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:23.972 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:24.011 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:24.018 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:24.018 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:24.018 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:24.018 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:24.018 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:24.018 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:24.018 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:24.018 [WARN - kudu-nio-6] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:24.018 [DEBUG - kudu-nio-6] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=188), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [141ms] refreshing cache from master, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [181ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [188ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [188ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [188ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [188ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@38442059(state=PENDING, result=null, callback=(continuation of Deferred@2037256055 after retry RPC after error@1154293823) -> (continuation of Deferred@101783147 after retry RPC after error@1345266222) -> (continuation of Deferred@1496550123 after retry RPC after error@1951569194) -> (continuation of Deferred@2063492997 after retry RPC after error@1407784102) -> (continuation of Deferred@373864674 after retry RPC after error@1993573572) -> (continuation of Deferred@2132281300 after retry RPC after error@1691312309) -> (continuation of Deferred@924015462 after retry RPC after error@1561074916), errback=(continuation of Deferred@2037256055 after retry RPC after error@1154293823) -> (continuation of Deferred@101783147 after retry RPC after error@1345266222) -> (continuation of Deferred@1496550123 after retry RPC after error@1951569194) -> (continuation of Deferred@2063492997 after retry RPC after error@1407784102) -> (continuation of Deferred@373864674 after retry RPC after error@1993573572) -> (continuation of Deferred@2132281300 after retry RPC after error@1691312309) -> (continuation of Deferred@924015462 after retry RPC after error@1561074916)))
20:28:24.019 [DEBUG - kudu-nio-6] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=188), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [141ms] refreshing cache from master, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [181ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [188ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [188ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [188ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [188ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@38442059(state=PENDING, result=null, callback=(continuation of Deferred@2037256055 after retry RPC after error@1154293823) -> (continuation of Deferred@101783147 after retry RPC after error@1345266222) -> (continuation of Deferred@1496550123 after retry RPC after error@1951569194) -> (continuation of Deferred@2063492997 after retry RPC after error@1407784102) -> (continuation of Deferred@373864674 after retry RPC after error@1993573572) -> (continuation of Deferred@2132281300 after retry RPC after error@1691312309) -> (continuation of Deferred@924015462 after retry RPC after error@1561074916), errback=(continuation of Deferred@2037256055 after retry RPC after error@1154293823) -> (continuation of Deferred@101783147 after retry RPC after error@1345266222) -> (continuation of Deferred@1496550123 after retry RPC after error@1951569194) -> (continuation of Deferred@2063492997 after retry RPC after error@1407784102) -> (continuation of Deferred@373864674 after retry RPC after error@1993573572) -> (continuation of Deferred@2132281300 after retry RPC after error@1691312309) -> (continuation of Deferred@924015462 after retry RPC after error@1561074916)))
20:28:24.030 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:24.030 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:24.031 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:24.031 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:24.031 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:24.031 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:24.031 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:24.031 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:24.031 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:24.051 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:24.051 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:24.052 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:24.052 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:24.052 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:24.052 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:24.052 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:24.052 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:24.052 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:24.071 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:24.071 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:24.071 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:24.071 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:24.071 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:24.071 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:24.071 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:24.072 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:24.072 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:24.072 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:24.072 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:24.073 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:24.091 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:24.091 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:24.091 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:24.091 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:24.091 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:24.091 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:24.091 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:24.091 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:24.092 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:24.110 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:24.111 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:24.111 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:24.111 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:24.111 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:24.112 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:24.112 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:24.112 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:24.113 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:24.130 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:24.131 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:24.131 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:24.131 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:24.131 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:24.131 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:24.131 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:24.131 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:24.132 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:24.191 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:24.191 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:24.191 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:24.191 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:24.191 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:24.191 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:24.191 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:24.191 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:24.192 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:24.193 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [43ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [83ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, deferred=Deferred@693062809(state=PENDING, result=null, callback=(continuation of Deferred@1623611813 after retry RPC after error@1582004359) -> (continuation of Deferred@96184289 after retry RPC after error@1455622219) -> (continuation of Deferred@68843591 after retry RPC after error@17868547) -> (continuation of Deferred@1498188911 after retry RPC after error@394479197) -> (continuation of Deferred@1074580650 after retry RPC after error@1151353623) -> (continuation of Deferred@2033356824 after retry RPC after error@1947111629), errback=(continuation of Deferred@1623611813 after retry RPC after error@1582004359) -> (continuation of Deferred@96184289 after retry RPC after error@1455622219) -> (continuation of Deferred@68843591 after retry RPC after error@17868547) -> (continuation of Deferred@1498188911 after retry RPC after error@394479197) -> (continuation of Deferred@1074580650 after retry RPC after error@1151353623) -> (continuation of Deferred@2033356824 after retry RPC after error@1947111629)))
20:28:24.193 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [43ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [83ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, deferred=Deferred@693062809(state=PENDING, result=null, callback=(continuation of Deferred@1623611813 after retry RPC after error@1582004359) -> (continuation of Deferred@96184289 after retry RPC after error@1455622219) -> (continuation of Deferred@68843591 after retry RPC after error@17868547) -> (continuation of Deferred@1498188911 after retry RPC after error@394479197) -> (continuation of Deferred@1074580650 after retry RPC after error@1151353623) -> (continuation of Deferred@2033356824 after retry RPC after error@1947111629), errback=(continuation of Deferred@1623611813 after retry RPC after error@1582004359) -> (continuation of Deferred@96184289 after retry RPC after error@1455622219) -> (continuation of Deferred@68843591 after retry RPC after error@17868547) -> (continuation of Deferred@1498188911 after retry RPC after error@394479197) -> (continuation of Deferred@1074580650 after retry RPC after error@1151353623) -> (continuation of Deferred@2033356824 after retry RPC after error@1947111629)))
20:28:24.210 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:24.211 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:24.211 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:24.211 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:24.211 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:24.211 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:24.211 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:24.211 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:24.212 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:24.230 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:24.231 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:24.231 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:24.231 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:24.231 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:24.231 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:24.231 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:24.232 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:24.232 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:24.250 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:24.251 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:24.251 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:24.251 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:24.251 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:24.251 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:24.251 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:24.251 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:24.252 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:24.270 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:24.270 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:24.271 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:24.271 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:24.271 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:24.271 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:24.271 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:24.271 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:24.272 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:24.291 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:24.291 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:24.292 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:24.292 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:24.292 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:24.292 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:24.292 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:24.292 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:24.292 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:24.331 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:24.331 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:24.331 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:24.332 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:24.332 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:24.332 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:24.332 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:24.332 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:24.333 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:24.391 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:24.391 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:24.392 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:24.392 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:24.392 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:24.392 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:24.392 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:24.392 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:24.393 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:24.471 [DEBUG - pool-40-thread-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=260), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [123ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [181ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [183ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@1022553451(state=PENDING, result=null, callback=(continuation of Deferred@588995384 after retry RPC after error@844667849) -> (continuation of Deferred@656511837 after retry RPC after error@211008359) -> (continuation of Deferred@1974002518 after retry RPC after error@64530681) -> (continuation of Deferred@675834020 after retry RPC after error@965846601) -> (continuation of Deferred@194072702 after retry RPC after error@1304997525) -> (continuation of Deferred@648999616 after retry RPC after error@305769394) -> (continuation of Deferred@329911083 after retry RPC after error@463385549), errback=(continuation of Deferred@588995384 after retry RPC after error@844667849) -> (continuation of Deferred@656511837 after retry RPC after error@211008359) -> (continuation of Deferred@1974002518 after retry RPC after error@64530681) -> (continuation of Deferred@675834020 after retry RPC after error@965846601) -> (continuation of Deferred@194072702 after retry RPC after error@1304997525) -> (continuation of Deferred@648999616 after retry RPC after error@305769394) -> (continuation of Deferred@329911083 after retry RPC after error@463385549)))
20:28:24.471 [WARN - pool-40-thread-1] (HashedWheelTimer.java:716) An exception was thrown by TimerTask.
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713)
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701)
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:787)
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501)
at java.base/java.lang.Thread.run(Thread.java:840)
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713)
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701)
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:787)
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501)
at java.base/java.lang.Thread.run(Thread.java:840)
20:28:24.491 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:24.491 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:24.491 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:24.492 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:24.492 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:24.492 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:24.492 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:24.492 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:24.492 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:24.511 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:24.511 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:24.512 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:24.512 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:24.512 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:24.512 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:24.512 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:24.512 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:24.513 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:24.531 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:24.531 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:24.531 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:24.532 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:24.532 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:24.532 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:24.532 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:24.532 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:24.532 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:24.551 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:24.551 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:24.551 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:24.551 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:24.551 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:24.552 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:24.552 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:24.552 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:24.552 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:24.571 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:24.571 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:24.571 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:24.571 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:24.571 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:24.571 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:24.572 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:24.572 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:24.573 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:24.610 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:24.611 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:24.611 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:24.611 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:24.611 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:24.611 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:24.611 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:24.611 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:24.612 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:24.671 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:24.671 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:24.671 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:24.671 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:24.671 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:24.671 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:24.672 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:24.672 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:24.672 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:24.731 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:24.731 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:24.732 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:24.732 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:24.732 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:24.732 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:24.732 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:24.732 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:24.732 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:24.733 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [241ms] refreshing cache from master, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, deferred=Deferred@1922347582(state=PENDING, result=null, callback=(continuation of Deferred@1152295957 after retry RPC after error@1954980620) -> (continuation of Deferred@529850522 after retry RPC after error@1212589076) -> (continuation of Deferred@2046377733 after retry RPC after error@950435317) -> (continuation of Deferred@9361995 after retry RPC after error@2098571338) -> (continuation of Deferred@1294250490 after retry RPC after error@1181099134) -> (continuation of Deferred@818737787 after retry RPC after error@1615547344) -> (continuation of Deferred@143166786 after retry RPC after error@2142903801), errback=(continuation of Deferred@1152295957 after retry RPC after error@1954980620) -> (continuation of Deferred@529850522 after retry RPC after error@1212589076) -> (continuation of Deferred@2046377733 after retry RPC after error@950435317) -> (continuation of Deferred@9361995 after retry RPC after error@2098571338) -> (continuation of Deferred@1294250490 after retry RPC after error@1181099134) -> (continuation of Deferred@818737787 after retry RPC after error@1615547344) -> (continuation of Deferred@143166786 after retry RPC after error@2142903801)))
20:28:24.733 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [241ms] refreshing cache from master, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, deferred=Deferred@1922347582(state=PENDING, result=null, callback=(continuation of Deferred@1152295957 after retry RPC after error@1954980620) -> (continuation of Deferred@529850522 after retry RPC after error@1212589076) -> (continuation of Deferred@2046377733 after retry RPC after error@950435317) -> (continuation of Deferred@9361995 after retry RPC after error@2098571338) -> (continuation of Deferred@1294250490 after retry RPC after error@1181099134) -> (continuation of Deferred@818737787 after retry RPC after error@1615547344) -> (continuation of Deferred@143166786 after retry RPC after error@2142903801), errback=(continuation of Deferred@1152295957 after retry RPC after error@1954980620) -> (continuation of Deferred@529850522 after retry RPC after error@1212589076) -> (continuation of Deferred@2046377733 after retry RPC after error@950435317) -> (continuation of Deferred@9361995 after retry RPC after error@2098571338) -> (continuation of Deferred@1294250490 after retry RPC after error@1181099134) -> (continuation of Deferred@818737787 after retry RPC after error@1615547344) -> (continuation of Deferred@143166786 after retry RPC after error@2142903801)))
20:28:24.751 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:24.751 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:24.752 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:24.752 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:24.752 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:24.752 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:24.752 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:24.752 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:24.753 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:24.771 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:24.771 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:24.771 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:24.771 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:24.772 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:24.772 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:24.772 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:24.772 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:24.772 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:24.791 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:24.791 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:24.791 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:24.791 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:24.792 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:24.792 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:24.792 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:24.792 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:24.793 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:24.811 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:24.811 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:24.811 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:24.811 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:24.811 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:24.811 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:24.811 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:24.811 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:24.812 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:24.830 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:24.831 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:24.831 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:24.831 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:24.831 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:24.831 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:24.831 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:24.831 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:24.831 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:24.831 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:24.832 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:24.832 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:24.870 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:24.871 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:24.871 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:24.871 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:24.871 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:24.871 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:24.871 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:24.871 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:24.872 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:24.931 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:24.932 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:24.932 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:24.932 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:24.932 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:24.932 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:24.932 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:24.932 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:24.933 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:24.934 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [20ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [40ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [60ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [80ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [119ms] refreshing cache from master, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [120ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [120ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [121ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [179ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@1249559501(state=PENDING, result=null, callback=(continuation of Deferred@897190268 after retry RPC after error@1971815228) -> (continuation of Deferred@1311010770 after retry RPC after error@1390193008) -> (continuation of Deferred@923298750 after retry RPC after error@2128580608) -> (continuation of Deferred@333986550 after retry RPC after error@608732882) -> (continuation of Deferred@1349078142 after retry RPC after error@1387563491) -> (continuation of Deferred@14231241 after retry RPC after error@1623678486), errback=(continuation of Deferred@897190268 after retry RPC after error@1971815228) -> (continuation of Deferred@1311010770 after retry RPC after error@1390193008) -> (continuation of Deferred@923298750 after retry RPC after error@2128580608) -> (continuation of Deferred@333986550 after retry RPC after error@608732882) -> (continuation of Deferred@1349078142 after retry RPC after error@1387563491) -> (continuation of Deferred@14231241 after retry RPC after error@1623678486)))
20:28:24.934 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [20ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [40ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [60ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [80ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [119ms] refreshing cache from master, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [120ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [120ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [121ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [179ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@1249559501(state=PENDING, result=null, callback=(continuation of Deferred@897190268 after retry RPC after error@1971815228) -> (continuation of Deferred@1311010770 after retry RPC after error@1390193008) -> (continuation of Deferred@923298750 after retry RPC after error@2128580608) -> (continuation of Deferred@333986550 after retry RPC after error@608732882) -> (continuation of Deferred@1349078142 after retry RPC after error@1387563491) -> (continuation of Deferred@14231241 after retry RPC after error@1623678486), errback=(continuation of Deferred@897190268 after retry RPC after error@1971815228) -> (continuation of Deferred@1311010770 after retry RPC after error@1390193008) -> (continuation of Deferred@923298750 after retry RPC after error@2128580608) -> (continuation of Deferred@333986550 after retry RPC after error@608732882) -> (continuation of Deferred@1349078142 after retry RPC after error@1387563491) -> (continuation of Deferred@14231241 after retry RPC after error@1623678486)))
20:28:24.950 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:24.950 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:24.951 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:24.951 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:24.951 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:24.951 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:24.951 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:24.951 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:24.952 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:24.971 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:24.971 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:24.972 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:24.972 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:24.972 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:24.972 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:24.972 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:24.972 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:24.973 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:24.991 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:24.991 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:24.991 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:24.991 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:24.992 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:24.992 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:24.992 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:24.992 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:24.992 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:25.011 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:25.011 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:25.011 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:25.011 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:25.012 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:25.012 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:25.012 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:25.012 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:25.013 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:25.030 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:25.031 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:25.031 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:25.031 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:25.031 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:25.031 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:25.031 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:25.031 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:25.032 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:25.070 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:25.071 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:25.071 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:25.071 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:25.071 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:25.071 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:25.071 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:25.071 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:25.073 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:25.090 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:25.090 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:25.091 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:25.091 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:25.091 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:25.091 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:25.091 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:25.091 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:25.092 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:25.092 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=141), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [23ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [63ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [123ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, deferred=Deferred@1174147528(state=PENDING, result=null, callback=(continuation of Deferred@521261199 after retry RPC after error@1512388149) -> (continuation of Deferred@36227380 after retry RPC after error@662694632) -> (continuation of Deferred@914084640 after retry RPC after error@1167904580) -> (continuation of Deferred@28958634 after retry RPC after error@1497976287) -> (continuation of Deferred@834893573 after retry RPC after error@302413974) -> (continuation of Deferred@1561178611 after retry RPC after error@1312368696), errback=(continuation of Deferred@521261199 after retry RPC after error@1512388149) -> (continuation of Deferred@36227380 after retry RPC after error@662694632) -> (continuation of Deferred@914084640 after retry RPC after error@1167904580) -> (continuation of Deferred@28958634 after retry RPC after error@1497976287) -> (continuation of Deferred@834893573 after retry RPC after error@302413974) -> (continuation of Deferred@1561178611 after retry RPC after error@1312368696)))
20:28:25.092 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=141), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [23ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [63ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [123ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, deferred=Deferred@1174147528(state=PENDING, result=null, callback=(continuation of Deferred@521261199 after retry RPC after error@1512388149) -> (continuation of Deferred@36227380 after retry RPC after error@662694632) -> (continuation of Deferred@914084640 after retry RPC after error@1167904580) -> (continuation of Deferred@28958634 after retry RPC after error@1497976287) -> (continuation of Deferred@834893573 after retry RPC after error@302413974) -> (continuation of Deferred@1561178611 after retry RPC after error@1312368696), errback=(continuation of Deferred@521261199 after retry RPC after error@1512388149) -> (continuation of Deferred@36227380 after retry RPC after error@662694632) -> (continuation of Deferred@914084640 after retry RPC after error@1167904580) -> (continuation of Deferred@28958634 after retry RPC after error@1497976287) -> (continuation of Deferred@834893573 after retry RPC after error@302413974) -> (continuation of Deferred@1561178611 after retry RPC after error@1312368696)))
20:28:25.111 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:25.111 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:25.112 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:25.112 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:25.112 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:25.112 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:25.112 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:25.112 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:25.113 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:25.131 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:25.131 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:25.131 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:25.131 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:25.132 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:25.132 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:25.132 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:25.132 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:25.132 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:25.151 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:25.151 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:25.151 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:25.151 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:25.151 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:25.152 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:25.152 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:25.152 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:25.152 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:25.171 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:25.171 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:25.171 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:25.171 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:25.171 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:25.171 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:25.171 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:25.171 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:25.172 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:25.190 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:25.191 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:25.191 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:25.191 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:25.191 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:25.191 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:25.191 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:25.191 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:25.192 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:25.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:28:25.212620 4132 master_runner.cc:186] Error getting master registration for 127.3.46.61:39407: Network error: Client connection negotiation failed: client connection to 127.3.46.61:39407: connect: Connection refused (error 111),
20:28:25.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:28:25.212841 4132 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
20:28:25.230 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:25.231 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:25.231 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:25.231 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:25.231 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:25.231 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:25.232 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:25.232 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:25.233 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:25.291 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:25.291 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:25.291 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:25.291 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:25.291 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:25.291 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:25.291 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:25.291 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:25.292 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:25.292 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=181), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [20ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [40ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [60ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [80ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [119ms] refreshing cache from master, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [120ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [179ms] refreshing cache from master, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [180ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [180ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [181ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@1453803706(state=PENDING, result=null, callback=(continuation of Deferred@1261072811 after retry RPC after error@1940593051) -> (continuation of Deferred@811420906 after retry RPC after error@1548951747) -> (continuation of Deferred@1147971019 after retry RPC after error@1864375068) -> (continuation of Deferred@893394958 after retry RPC after error@1892703069) -> (continuation of Deferred@1362356849 after retry RPC after error@1042609072) -> (continuation of Deferred@181462058 after retry RPC after error@2015906974), errback=(continuation of Deferred@1261072811 after retry RPC after error@1940593051) -> (continuation of Deferred@811420906 after retry RPC after error@1548951747) -> (continuation of Deferred@1147971019 after retry RPC after error@1864375068) -> (continuation of Deferred@893394958 after retry RPC after error@1892703069) -> (continuation of Deferred@1362356849 after retry RPC after error@1042609072) -> (continuation of Deferred@181462058 after retry RPC after error@2015906974)))
20:28:25.292 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=181), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [20ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [40ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [60ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [80ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [119ms] refreshing cache from master, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [120ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [179ms] refreshing cache from master, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [180ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [180ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [181ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@1453803706(state=PENDING, result=null, callback=(continuation of Deferred@1261072811 after retry RPC after error@1940593051) -> (continuation of Deferred@811420906 after retry RPC after error@1548951747) -> (continuation of Deferred@1147971019 after retry RPC after error@1864375068) -> (continuation of Deferred@893394958 after retry RPC after error@1892703069) -> (continuation of Deferred@1362356849 after retry RPC after error@1042609072) -> (continuation of Deferred@181462058 after retry RPC after error@2015906974), errback=(continuation of Deferred@1261072811 after retry RPC after error@1940593051) -> (continuation of Deferred@811420906 after retry RPC after error@1548951747) -> (continuation of Deferred@1147971019 after retry RPC after error@1864375068) -> (continuation of Deferred@893394958 after retry RPC after error@1892703069) -> (continuation of Deferred@1362356849 after retry RPC after error@1042609072) -> (continuation of Deferred@181462058 after retry RPC after error@2015906974)))
20:28:25.310 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:25.311 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:25.311 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:25.311 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:25.311 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:25.311 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:25.311 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:25.311 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:25.311 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:25.311 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:25.312 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:25.312 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:25.330 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:25.330 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:25.331 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:25.331 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:25.331 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:25.331 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:25.331 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:25.331 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:25.331 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:25.351 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:25.351 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:25.352 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:25.352 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:25.352 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:25.352 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:25.352 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:25.352 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:25.353 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:25.371 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:25.371 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:25.371 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:25.371 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:25.371 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:25.371 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:25.372 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:25.372 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:25.372 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:25.391 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:25.391 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:25.391 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:25.391 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:25.391 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:25.391 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:25.391 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:25.391 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:25.392 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:25.430 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:25.431 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:25.431 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:25.431 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:25.431 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:25.431 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:25.431 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:25.431 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:25.432 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:25.450 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:25.451 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:25.451 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:25.451 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:25.451 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:25.451 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:25.451 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:25.451 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:25.452 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:25.452 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=141), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [0ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [1ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [20ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [80ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [119ms] refreshing cache from master, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [120ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [120ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [121ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [139ms] refreshing cache from master, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [140ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [140ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [141ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, deferred=Deferred@472002820(state=PENDING, result=null, callback=(continuation of Deferred@133377502 after retry RPC after error@1932713220) -> (continuation of Deferred@179002003 after retry RPC after error@118288595) -> (continuation of Deferred@420454449 after retry RPC after error@2091943145) -> (continuation of Deferred@740242990 after retry RPC after error@1282785558) -> (continuation of Deferred@673181313 after retry RPC after error@1349562535) -> (continuation of Deferred@93012673 after retry RPC after error@1729388055), errback=(continuation of Deferred@133377502 after retry RPC after error@1932713220) -> (continuation of Deferred@179002003 after retry RPC after error@118288595) -> (continuation of Deferred@420454449 after retry RPC after error@2091943145) -> (continuation of Deferred@740242990 after retry RPC after error@1282785558) -> (continuation of Deferred@673181313 after retry RPC after error@1349562535) -> (continuation of Deferred@93012673 after retry RPC after error@1729388055)))
20:28:25.452 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=141), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [0ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [1ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [20ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [80ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [119ms] refreshing cache from master, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [120ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [120ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [121ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [139ms] refreshing cache from master, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [140ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [140ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [141ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, deferred=Deferred@472002820(state=PENDING, result=null, callback=(continuation of Deferred@133377502 after retry RPC after error@1932713220) -> (continuation of Deferred@179002003 after retry RPC after error@118288595) -> (continuation of Deferred@420454449 after retry RPC after error@2091943145) -> (continuation of Deferred@740242990 after retry RPC after error@1282785558) -> (continuation of Deferred@673181313 after retry RPC after error@1349562535) -> (continuation of Deferred@93012673 after retry RPC after error@1729388055), errback=(continuation of Deferred@133377502 after retry RPC after error@1932713220) -> (continuation of Deferred@179002003 after retry RPC after error@118288595) -> (continuation of Deferred@420454449 after retry RPC after error@2091943145) -> (continuation of Deferred@740242990 after retry RPC after error@1282785558) -> (continuation of Deferred@673181313 after retry RPC after error@1349562535) -> (continuation of Deferred@93012673 after retry RPC after error@1729388055)))
20:28:25.470 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:25.470 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:25.471 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:25.471 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:25.471 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:25.471 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:25.471 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:25.471 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:25.472 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:25.491 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:25.491 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:25.492 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:25.492 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:25.492 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:25.492 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:25.492 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:25.492 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:25.492 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:25.511 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:25.511 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:25.512 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:25.512 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:25.512 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:25.512 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:25.512 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:25.512 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:25.512 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:25.531 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:25.531 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:25.531 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:25.531 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:25.531 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:25.531 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:25.531 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:25.531 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:25.532 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:25.551 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:25.551 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:25.551 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:25.551 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:25.551 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:25.551 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:25.551 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:25.551 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:25.552 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:25.570 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:25.571 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:25.571 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:25.571 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:25.571 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:25.571 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:25.571 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:25.571 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:25.572 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:25.590 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:25.590 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:25.591 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:25.591 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:25.591 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:25.591 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:25.591 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:25.591 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:25.592 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:25.690 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:25.691 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:25.691 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:25.691 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:25.691 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:25.691 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:25.691 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:25.691 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:25.692 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:25.692 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [222ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@164199946(state=PENDING, result=null, callback=(continuation of Deferred@700720751 after retry RPC after error@1075571386) -> (continuation of Deferred@1678107177 after retry RPC after error@645855629) -> (continuation of Deferred@771839091 after retry RPC after error@827340083) -> (continuation of Deferred@2047843738 after retry RPC after error@728539591) -> (continuation of Deferred@258656789 after retry RPC after error@1067683686) -> (continuation of Deferred@969291333 after retry RPC after error@2011875875) -> (continuation of Deferred@772381865 after retry RPC after error@225636017), errback=(continuation of Deferred@700720751 after retry RPC after error@1075571386) -> (continuation of Deferred@1678107177 after retry RPC after error@645855629) -> (continuation of Deferred@771839091 after retry RPC after error@827340083) -> (continuation of Deferred@2047843738 after retry RPC after error@728539591) -> (continuation of Deferred@258656789 after retry RPC after error@1067683686) -> (continuation of Deferred@969291333 after retry RPC after error@2011875875) -> (continuation of Deferred@772381865 after retry RPC after error@225636017)))
20:28:25.692 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [222ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@164199946(state=PENDING, result=null, callback=(continuation of Deferred@700720751 after retry RPC after error@1075571386) -> (continuation of Deferred@1678107177 after retry RPC after error@645855629) -> (continuation of Deferred@771839091 after retry RPC after error@827340083) -> (continuation of Deferred@2047843738 after retry RPC after error@728539591) -> (continuation of Deferred@258656789 after retry RPC after error@1067683686) -> (continuation of Deferred@969291333 after retry RPC after error@2011875875) -> (continuation of Deferred@772381865 after retry RPC after error@225636017), errback=(continuation of Deferred@700720751 after retry RPC after error@1075571386) -> (continuation of Deferred@1678107177 after retry RPC after error@645855629) -> (continuation of Deferred@771839091 after retry RPC after error@827340083) -> (continuation of Deferred@2047843738 after retry RPC after error@728539591) -> (continuation of Deferred@258656789 after retry RPC after error@1067683686) -> (continuation of Deferred@969291333 after retry RPC after error@2011875875) -> (continuation of Deferred@772381865 after retry RPC after error@225636017)))
20:28:25.710 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:25.711 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:25.711 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:25.711 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:25.711 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:25.711 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:25.711 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:25.711 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:25.712 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:25.730 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:25.731 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:25.731 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:25.731 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:25.731 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:25.731 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:25.731 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:25.731 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:25.732 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:25.751 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:25.751 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:25.751 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:25.751 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:25.751 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:25.751 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:25.752 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:25.752 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:25.752 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:25.771 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:25.771 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:25.771 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:25.771 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:25.771 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:25.771 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:25.771 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:25.772 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:25.772 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:25.790 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:25.791 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:25.791 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:25.791 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:25.791 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:25.791 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:25.791 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:25.791 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:25.792 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:25.830 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:25.830 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:25.831 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:25.831 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:25.831 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:25.831 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:25.831 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:25.831 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:25.832 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:25.870 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:25.870 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:25.871 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:25.871 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:25.871 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:25.871 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:25.871 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:25.871 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:25.872 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:25.910 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:25.910 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:25.911 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:25.911 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:25.911 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:25.911 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:25.911 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:25.911 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:25.912 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:25.912 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=201), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [40ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@760357790(state=PENDING, result=null, callback=(continuation of Deferred@774217324 after retry RPC after error@1042849371) -> (continuation of Deferred@1593937968 after retry RPC after error@1676875421) -> (continuation of Deferred@1638347377 after retry RPC after error@461877848) -> (continuation of Deferred@1221861640 after retry RPC after error@862486897) -> (continuation of Deferred@1803815383 after retry RPC after error@1303868301) -> (continuation of Deferred@604165970 after retry RPC after error@1393994154) -> (continuation of Deferred@404211886 after retry RPC after error@1850407738), errback=(continuation of Deferred@774217324 after retry RPC after error@1042849371) -> (continuation of Deferred@1593937968 after retry RPC after error@1676875421) -> (continuation of Deferred@1638347377 after retry RPC after error@461877848) -> (continuation of Deferred@1221861640 after retry RPC after error@862486897) -> (continuation of Deferred@1803815383 after retry RPC after error@1303868301) -> (continuation of Deferred@604165970 after retry RPC after error@1393994154) -> (continuation of Deferred@404211886 after retry RPC after error@1850407738)))
20:28:25.912 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=201), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [40ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@760357790(state=PENDING, result=null, callback=(continuation of Deferred@774217324 after retry RPC after error@1042849371) -> (continuation of Deferred@1593937968 after retry RPC after error@1676875421) -> (continuation of Deferred@1638347377 after retry RPC after error@461877848) -> (continuation of Deferred@1221861640 after retry RPC after error@862486897) -> (continuation of Deferred@1803815383 after retry RPC after error@1303868301) -> (continuation of Deferred@604165970 after retry RPC after error@1393994154) -> (continuation of Deferred@404211886 after retry RPC after error@1850407738), errback=(continuation of Deferred@774217324 after retry RPC after error@1042849371) -> (continuation of Deferred@1593937968 after retry RPC after error@1676875421) -> (continuation of Deferred@1638347377 after retry RPC after error@461877848) -> (continuation of Deferred@1221861640 after retry RPC after error@862486897) -> (continuation of Deferred@1803815383 after retry RPC after error@1303868301) -> (continuation of Deferred@604165970 after retry RPC after error@1393994154) -> (continuation of Deferred@404211886 after retry RPC after error@1850407738)))
20:28:25.931 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:25.931 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:25.932 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:25.932 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:25.932 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:25.932 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:25.932 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:25.932 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:25.933 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:25.951 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:25.951 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:25.952 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:25.952 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:25.952 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:25.952 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:25.952 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:25.952 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:25.952 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:25.971 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:25.971 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:25.971 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:25.972 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:25.972 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:25.972 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:25.972 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:25.972 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:25.972 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:25.991 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:25.991 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:25.991 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:25.991 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:25.991 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:25.991 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:25.991 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:25.991 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:25.992 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:26.010 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:26.011 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:26.011 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:26.011 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:26.011 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:26.011 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:26.011 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:26.011 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:26.012 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:26.030 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:26.030 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:26.031 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:26.031 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:26.031 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:26.031 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:26.031 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:26.031 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:26.032 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:26.090 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:26.091 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:26.091 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:26.091 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:26.091 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:26.091 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:26.091 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:26.091 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:26.092 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:26.110 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:26.110 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:26.111 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:26.111 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:26.111 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:26.111 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:26.111 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:26.111 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:26.112 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:26.112 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=181), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [60ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [80ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [99ms] refreshing cache from master, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [100ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [100ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [101ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [159ms] refreshing cache from master, [159ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [159ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [160ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [160ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [161ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [179ms] refreshing cache from master, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [180ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [180ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [181ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@2045822111(state=PENDING, result=null, callback=(continuation of Deferred@777748420 after retry RPC after error@2028987654) -> (continuation of Deferred@1111603241 after retry RPC after error@1038764226) -> (continuation of Deferred@1211803108 after retry RPC after error@1333440048) -> (continuation of Deferred@3528454 after retry RPC after error@1870017434) -> (continuation of Deferred@1920904725 after retry RPC after error@1760527684) -> (continuation of Deferred@1179447992 after retry RPC after error@410575624) -> (continuation of Deferred@535947284 after retry RPC after error@195074827), errback=(continuation of Deferred@777748420 after retry RPC after error@2028987654) -> (continuation of Deferred@1111603241 after retry RPC after error@1038764226) -> (continuation of Deferred@1211803108 after retry RPC after error@1333440048) -> (continuation of Deferred@3528454 after retry RPC after error@1870017434) -> (continuation of Deferred@1920904725 after retry RPC after error@1760527684) -> (continuation of Deferred@1179447992 after retry RPC after error@410575624) -> (continuation of Deferred@535947284 after retry RPC after error@195074827)))
20:28:26.113 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=181), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [60ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [80ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [99ms] refreshing cache from master, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [100ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [100ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [101ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [159ms] refreshing cache from master, [159ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [159ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [160ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [160ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [161ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [179ms] refreshing cache from master, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [180ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [180ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [181ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@2045822111(state=PENDING, result=null, callback=(continuation of Deferred@777748420 after retry RPC after error@2028987654) -> (continuation of Deferred@1111603241 after retry RPC after error@1038764226) -> (continuation of Deferred@1211803108 after retry RPC after error@1333440048) -> (continuation of Deferred@3528454 after retry RPC after error@1870017434) -> (continuation of Deferred@1920904725 after retry RPC after error@1760527684) -> (continuation of Deferred@1179447992 after retry RPC after error@410575624) -> (continuation of Deferred@535947284 after retry RPC after error@195074827), errback=(continuation of Deferred@777748420 after retry RPC after error@2028987654) -> (continuation of Deferred@1111603241 after retry RPC after error@1038764226) -> (continuation of Deferred@1211803108 after retry RPC after error@1333440048) -> (continuation of Deferred@3528454 after retry RPC after error@1870017434) -> (continuation of Deferred@1920904725 after retry RPC after error@1760527684) -> (continuation of Deferred@1179447992 after retry RPC after error@410575624) -> (continuation of Deferred@535947284 after retry RPC after error@195074827)))
20:28:26.131 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:26.131 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:26.131 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:26.131 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:26.132 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:26.132 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:26.132 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:26.132 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:26.132 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:26.151 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:26.151 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:26.151 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:26.151 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:26.151 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:26.151 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:26.151 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:26.151 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:26.152 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:26.170 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:26.171 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:26.171 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:26.171 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:26.171 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:26.171 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:26.171 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:26.171 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:26.172 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:26.190 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:26.191 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:26.191 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:26.191 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:26.191 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:26.191 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:26.191 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:26.191 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:26.192 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:26.210 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:26.210 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:26.211 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:26.211 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:26.211 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:26.211 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:26.211 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:26.211 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:26.212 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:26.250 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:26.250 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:26.251 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:26.251 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:26.251 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:26.251 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:26.251 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:26.251 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:26.252 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:26.291 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:26.291 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:26.292 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:26.292 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:26.292 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:26.292 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:26.292 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:26.292 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:26.292 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:26.331 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:26.331 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:26.332 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:26.332 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:26.332 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:26.332 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:26.332 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:26.332 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:26.333 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:26.371 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:26.371 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:26.372 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:26.372 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:26.372 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:26.372 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:26.372 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:26.372 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:26.372 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:26.373 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=241), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [0ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [1ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [19ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [20ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [40ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [60ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [80ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [119ms] refreshing cache from master, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [120ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [120ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [121ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [241ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@59729151(state=PENDING, result=null, callback=(continuation of Deferred@1268298756 after retry RPC after error@479069306) -> (continuation of Deferred@2045134241 after retry RPC after error@20344574) -> (continuation of Deferred@260100680 after retry RPC after error@1665247766) -> (continuation of Deferred@675356818 after retry RPC after error@439544570) -> (continuation of Deferred@321485158 after retry RPC after error@2102812687) -> (continuation of Deferred@1555204743 after retry RPC after error@1373298385) -> (continuation of Deferred@165707717 after retry RPC after error@1207443694) -> (continuation of Deferred@265710628 after retry RPC after error@142415705), errback=(continuation of Deferred@1268298756 after retry RPC after error@479069306) -> (continuation of Deferred@2045134241 after retry RPC after error@20344574) -> (continuation of Deferred@260100680 after retry RPC after error@1665247766) -> (continuation of Deferred@675356818 after retry RPC after error@439544570) -> (continuation of Deferred@321485158 after retry RPC after error@2102812687) -> (continuation of Deferred@1555204743 after retry RPC after error@1373298385) -> (continuation of Deferred@165707717 after retry RPC after error@1207443694) -> (continuation of Deferred@265710628 after retry RPC after error@142415705)))
20:28:26.373 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=241), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [0ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [1ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [19ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [20ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [40ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [60ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [80ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [119ms] refreshing cache from master, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [120ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [120ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [121ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [241ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@59729151(state=PENDING, result=null, callback=(continuation of Deferred@1268298756 after retry RPC after error@479069306) -> (continuation of Deferred@2045134241 after retry RPC after error@20344574) -> (continuation of Deferred@260100680 after retry RPC after error@1665247766) -> (continuation of Deferred@675356818 after retry RPC after error@439544570) -> (continuation of Deferred@321485158 after retry RPC after error@2102812687) -> (continuation of Deferred@1555204743 after retry RPC after error@1373298385) -> (continuation of Deferred@165707717 after retry RPC after error@1207443694) -> (continuation of Deferred@265710628 after retry RPC after error@142415705), errback=(continuation of Deferred@1268298756 after retry RPC after error@479069306) -> (continuation of Deferred@2045134241 after retry RPC after error@20344574) -> (continuation of Deferred@260100680 after retry RPC after error@1665247766) -> (continuation of Deferred@675356818 after retry RPC after error@439544570) -> (continuation of Deferred@321485158 after retry RPC after error@2102812687) -> (continuation of Deferred@1555204743 after retry RPC after error@1373298385) -> (continuation of Deferred@165707717 after retry RPC after error@1207443694) -> (continuation of Deferred@265710628 after retry RPC after error@142415705)))
20:28:26.391 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:26.391 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:26.391 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:26.391 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:26.392 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:26.392 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:26.392 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:26.392 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:26.392 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:26.411 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:26.411 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:26.411 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:26.411 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:26.411 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:26.411 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:26.412 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:26.412 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:26.412 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:26.430 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:26.431 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:26.431 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:26.431 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:26.431 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:26.431 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:26.431 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:26.431 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:26.432 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:26.450 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:26.450 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:26.451 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:26.451 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:26.451 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:26.451 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:26.451 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:26.451 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:26.452 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:26.470 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:26.470 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:26.471 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:26.471 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:26.471 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:26.471 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:26.471 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:26.471 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:26.471 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:26.491 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:26.491 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:26.491 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:26.491 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:26.492 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:26.492 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:26.492 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:26.492 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:26.492 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:26.570 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:26.570 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:26.571 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:26.572 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:26.572 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:26.572 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:26.572 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:26.572 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:26.572 [WARN - kudu-nio-2] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:26.572 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=181), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [1ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [20ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [40ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [60ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [80ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [101ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [179ms] refreshing cache from master, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [181ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@1679550319(state=PENDING, result=null, callback=(continuation of Deferred@538361583 after retry RPC after error@1301192978) -> (continuation of Deferred@776603454 after retry RPC after error@366066951) -> (continuation of Deferred@10848404 after retry RPC after error@941309859) -> (continuation of Deferred@1175895145 after retry RPC after error@1580664873) -> (continuation of Deferred@1929038419 after retry RPC after error@335349128) -> (continuation of Deferred@1627068010 after retry RPC after error@107686940), errback=(continuation of Deferred@538361583 after retry RPC after error@1301192978) -> (continuation of Deferred@776603454 after retry RPC after error@366066951) -> (continuation of Deferred@10848404 after retry RPC after error@941309859) -> (continuation of Deferred@1175895145 after retry RPC after error@1580664873) -> (continuation of Deferred@1929038419 after retry RPC after error@335349128) -> (continuation of Deferred@1627068010 after retry RPC after error@107686940)))
20:28:26.572 [DEBUG - kudu-nio-2] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=181), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [1ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [20ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [40ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [60ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [80ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [101ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [179ms] refreshing cache from master, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [181ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@1679550319(state=PENDING, result=null, callback=(continuation of Deferred@538361583 after retry RPC after error@1301192978) -> (continuation of Deferred@776603454 after retry RPC after error@366066951) -> (continuation of Deferred@10848404 after retry RPC after error@941309859) -> (continuation of Deferred@1175895145 after retry RPC after error@1580664873) -> (continuation of Deferred@1929038419 after retry RPC after error@335349128) -> (continuation of Deferred@1627068010 after retry RPC after error@107686940), errback=(continuation of Deferred@538361583 after retry RPC after error@1301192978) -> (continuation of Deferred@776603454 after retry RPC after error@366066951) -> (continuation of Deferred@10848404 after retry RPC after error@941309859) -> (continuation of Deferred@1175895145 after retry RPC after error@1580664873) -> (continuation of Deferred@1929038419 after retry RPC after error@335349128) -> (continuation of Deferred@1627068010 after retry RPC after error@107686940)))
20:28:26.591 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:26.591 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:26.592 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:26.592 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:26.592 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:26.592 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:26.592 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:26.592 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:26.592 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:26.611 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:26.611 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:26.611 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:26.611 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:26.611 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:26.611 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:26.611 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:26.611 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:26.612 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:26.630 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:26.631 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:26.631 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:26.631 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:26.631 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:26.631 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:26.631 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:26.631 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:26.632 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:26.650 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:26.650 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:26.651 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:26.651 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:26.651 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:26.651 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:26.651 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:26.651 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:26.652 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:26.671 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:26.671 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:26.672 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:26.672 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:26.672 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:26.672 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:26.672 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:26.672 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:26.672 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:26.711 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:26.711 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:26.711 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:26.712 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:26.712 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:26.712 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:26.712 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:26.712 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:26.712 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:26.751 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:26.751 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:26.752 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:26.752 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:26.752 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:26.752 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:26.752 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:26.752 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:26.752 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:26.771 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:26.771 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:26.771 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:26.771 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:26.771 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:26.772 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:26.772 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:26.772 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:26.772 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:26.773 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=181), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [1ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [20ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [40ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [60ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [181ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@1143037564(state=PENDING, result=null, callback=(continuation of Deferred@1579860971 after retry RPC after error@158333513) -> (continuation of Deferred@1411721345 after retry RPC after error@439963335) -> (continuation of Deferred@302314480 after retry RPC after error@1682959947) -> (continuation of Deferred@424911715 after retry RPC after error@1229756920) -> (continuation of Deferred@2086828114 after retry RPC after error@1732570147) -> (continuation of Deferred@1669858067 after retry RPC after error@1311167939) -> (continuation of Deferred@560567485 after retry RPC after error@1636585799), errback=(continuation of Deferred@1579860971 after retry RPC after error@158333513) -> (continuation of Deferred@1411721345 after retry RPC after error@439963335) -> (continuation of Deferred@302314480 after retry RPC after error@1682959947) -> (continuation of Deferred@424911715 after retry RPC after error@1229756920) -> (continuation of Deferred@2086828114 after retry RPC after error@1732570147) -> (continuation of Deferred@1669858067 after retry RPC after error@1311167939) -> (continuation of Deferred@560567485 after retry RPC after error@1636585799)))
20:28:26.773 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=181), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [1ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [20ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [40ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [60ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [181ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@1143037564(state=PENDING, result=null, callback=(continuation of Deferred@1579860971 after retry RPC after error@158333513) -> (continuation of Deferred@1411721345 after retry RPC after error@439963335) -> (continuation of Deferred@302314480 after retry RPC after error@1682959947) -> (continuation of Deferred@424911715 after retry RPC after error@1229756920) -> (continuation of Deferred@2086828114 after retry RPC after error@1732570147) -> (continuation of Deferred@1669858067 after retry RPC after error@1311167939) -> (continuation of Deferred@560567485 after retry RPC after error@1636585799), errback=(continuation of Deferred@1579860971 after retry RPC after error@158333513) -> (continuation of Deferred@1411721345 after retry RPC after error@439963335) -> (continuation of Deferred@302314480 after retry RPC after error@1682959947) -> (continuation of Deferred@424911715 after retry RPC after error@1229756920) -> (continuation of Deferred@2086828114 after retry RPC after error@1732570147) -> (continuation of Deferred@1669858067 after retry RPC after error@1311167939) -> (continuation of Deferred@560567485 after retry RPC after error@1636585799)))
20:28:26.790 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:26.791 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:26.791 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:26.791 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:26.791 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:26.791 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:26.791 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:26.791 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:26.792 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:26.810 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:26.811 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:26.811 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:26.811 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:26.811 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:26.811 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:26.811 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:26.811 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:26.812 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:26.830 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:26.830 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:26.831 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:26.831 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:26.831 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:26.831 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:26.831 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:26.831 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:26.832 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:26.851 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:26.851 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:26.851 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:26.851 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:26.851 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:26.852 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:26.852 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:26.852 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:26.852 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:26.871 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:26.871 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:26.871 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:26.871 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:26.871 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:26.871 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:26.871 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:26.871 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:26.872 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:26.890 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:26.891 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:26.891 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:26.891 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:26.891 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:26.891 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:26.891 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:26.892 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:26.892 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:26.951 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:26.951 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:26.951 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:26.951 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:26.951 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:26.951 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:26.951 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:26.952 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:26.952 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:27.011 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:27.011 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:27.011 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:27.011 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:27.011 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:27.011 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:27.011 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:27.011 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:27.012 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:27.012 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=221), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [222ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@1826352441(state=PENDING, result=null, callback=(continuation of Deferred@780882358 after retry RPC after error@212209564) -> (continuation of Deferred@101188557 after retry RPC after error@889820176) -> (continuation of Deferred@1098606054 after retry RPC after error@2099071162) -> (continuation of Deferred@994917055 after retry RPC after error@1109229778) -> (continuation of Deferred@1481873698 after retry RPC after error@1335640586) -> (continuation of Deferred@492179844 after retry RPC after error@862132716) -> (continuation of Deferred@2008878123 after retry RPC after error@1520213188), errback=(continuation of Deferred@780882358 after retry RPC after error@212209564) -> (continuation of Deferred@101188557 after retry RPC after error@889820176) -> (continuation of Deferred@1098606054 after retry RPC after error@2099071162) -> (continuation of Deferred@994917055 after retry RPC after error@1109229778) -> (continuation of Deferred@1481873698 after retry RPC after error@1335640586) -> (continuation of Deferred@492179844 after retry RPC after error@862132716) -> (continuation of Deferred@2008878123 after retry RPC after error@1520213188)))
20:28:27.013 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=221), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [222ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@1826352441(state=PENDING, result=null, callback=(continuation of Deferred@780882358 after retry RPC after error@212209564) -> (continuation of Deferred@101188557 after retry RPC after error@889820176) -> (continuation of Deferred@1098606054 after retry RPC after error@2099071162) -> (continuation of Deferred@994917055 after retry RPC after error@1109229778) -> (continuation of Deferred@1481873698 after retry RPC after error@1335640586) -> (continuation of Deferred@492179844 after retry RPC after error@862132716) -> (continuation of Deferred@2008878123 after retry RPC after error@1520213188), errback=(continuation of Deferred@780882358 after retry RPC after error@212209564) -> (continuation of Deferred@101188557 after retry RPC after error@889820176) -> (continuation of Deferred@1098606054 after retry RPC after error@2099071162) -> (continuation of Deferred@994917055 after retry RPC after error@1109229778) -> (continuation of Deferred@1481873698 after retry RPC after error@1335640586) -> (continuation of Deferred@492179844 after retry RPC after error@862132716) -> (continuation of Deferred@2008878123 after retry RPC after error@1520213188)))
20:28:27.030 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:27.031 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:27.031 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:27.031 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:27.031 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:27.031 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:27.031 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:27.031 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:27.032 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:27.050 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:27.050 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:27.051 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:27.051 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:27.051 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:27.051 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:27.051 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:27.051 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:27.052 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:27.071 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:27.071 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:27.071 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:27.071 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:27.071 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:27.072 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:27.072 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:27.072 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:27.072 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:27.091 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:27.091 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:27.091 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:27.091 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:27.091 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:27.092 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:27.092 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:27.092 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:27.092 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:27.110 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:27.110 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:27.111 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:27.111 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:27.111 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:27.111 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:27.111 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:27.111 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:27.112 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:27.150 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:27.150 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:27.151 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:27.151 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:27.151 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:27.151 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:27.151 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:27.151 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:27.152 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:27.230 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:27.231 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:27.231 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:27.232 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:27.232 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:27.232 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:27.232 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:27.232 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:27.232 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:27.290 [DEBUG - pool-40-thread-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=259), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [202ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@2126020776(state=PENDING, result=null, callback=(continuation of Deferred@1988650556 after retry RPC after error@293313013) -> (continuation of Deferred@646906650 after retry RPC after error@470313635) -> (continuation of Deferred@688146072 after retry RPC after error@1694353020) -> (continuation of Deferred@984911402 after retry RPC after error@1698084525) -> (continuation of Deferred@1846348821 after retry RPC after error@1682565518) -> (continuation of Deferred@2056449779 after retry RPC after error@717335762) -> (continuation of Deferred@749363259 after retry RPC after error@117766585), errback=(continuation of Deferred@1988650556 after retry RPC after error@293313013) -> (continuation of Deferred@646906650 after retry RPC after error@470313635) -> (continuation of Deferred@688146072 after retry RPC after error@1694353020) -> (continuation of Deferred@984911402 after retry RPC after error@1698084525) -> (continuation of Deferred@1846348821 after retry RPC after error@1682565518) -> (continuation of Deferred@2056449779 after retry RPC after error@717335762) -> (continuation of Deferred@749363259 after retry RPC after error@117766585)))
20:28:27.291 [WARN - pool-40-thread-1] (HashedWheelTimer.java:716) An exception was thrown by TimerTask.
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713)
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701)
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:787)
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501)
at java.base/java.lang.Thread.run(Thread.java:840)
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713)
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701)
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:787)
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501)
at java.base/java.lang.Thread.run(Thread.java:840)
20:28:27.310 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:27.311 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:27.311 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:27.311 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:27.311 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:27.311 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:27.311 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:27.311 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:27.312 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:27.330 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:27.331 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:27.331 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:27.331 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:27.331 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:27.331 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:27.331 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:27.331 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:27.332 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:27.350 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:27.350 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:27.351 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:27.351 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:27.351 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:27.351 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:27.351 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:27.351 [INFO - pool-40-thread-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:27.352 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:27.371 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:27.371 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:27.371 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:27.371 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:27.372 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:27.372 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:27.372 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:27.372 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:27.372 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:27.391 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:27.391 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:27.391 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:27.391 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:27.391 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:27.391 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:27.392 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:27.392 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:27.392 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:27.411 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:27.411 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:27.412 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:27.412 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:27.412 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:27.412 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:27.412 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:27.413 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:27.413 [WARN - kudu-nio-4] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:27.471 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:27.471 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:27.472 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:27.472 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:27.472 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:27.472 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:27.472 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:27.472 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:27.473 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:27.511 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:27.511 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:27.512 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:27.512 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:27.512 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:27.512 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:27.512 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:27.512 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:27.512 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:27.513 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [103ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [161ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [201ms] refreshing cache from master, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [203ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@1239467867(state=PENDING, result=null, callback=(continuation of Deferred@813535901 after retry RPC after error@1895026947) -> (continuation of Deferred@1964341623 after retry RPC after error@981508753) -> (continuation of Deferred@1714759206 after retry RPC after error@144738346) -> (continuation of Deferred@1444976264 after retry RPC after error@1762337527) -> (continuation of Deferred@641740262 after retry RPC after error@715981020) -> (continuation of Deferred@2020300759 after retry RPC after error@1221054135) -> (continuation of Deferred@1867511124 after retry RPC after error@757506945), errback=(continuation of Deferred@813535901 after retry RPC after error@1895026947) -> (continuation of Deferred@1964341623 after retry RPC after error@981508753) -> (continuation of Deferred@1714759206 after retry RPC after error@144738346) -> (continuation of Deferred@1444976264 after retry RPC after error@1762337527) -> (continuation of Deferred@641740262 after retry RPC after error@715981020) -> (continuation of Deferred@2020300759 after retry RPC after error@1221054135) -> (continuation of Deferred@1867511124 after retry RPC after error@757506945)))
20:28:27.513 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [103ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [161ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [201ms] refreshing cache from master, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [203ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@1239467867(state=PENDING, result=null, callback=(continuation of Deferred@813535901 after retry RPC after error@1895026947) -> (continuation of Deferred@1964341623 after retry RPC after error@981508753) -> (continuation of Deferred@1714759206 after retry RPC after error@144738346) -> (continuation of Deferred@1444976264 after retry RPC after error@1762337527) -> (continuation of Deferred@641740262 after retry RPC after error@715981020) -> (continuation of Deferred@2020300759 after retry RPC after error@1221054135) -> (continuation of Deferred@1867511124 after retry RPC after error@757506945), errback=(continuation of Deferred@813535901 after retry RPC after error@1895026947) -> (continuation of Deferred@1964341623 after retry RPC after error@981508753) -> (continuation of Deferred@1714759206 after retry RPC after error@144738346) -> (continuation of Deferred@1444976264 after retry RPC after error@1762337527) -> (continuation of Deferred@641740262 after retry RPC after error@715981020) -> (continuation of Deferred@2020300759 after retry RPC after error@1221054135) -> (continuation of Deferred@1867511124 after retry RPC after error@757506945)))
20:28:27.531 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:27.531 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:27.531 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:27.531 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:27.531 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:27.531 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:27.531 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:27.532 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:27.532 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:27.551 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:27.551 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:27.551 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:27.551 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:27.551 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:27.551 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:27.551 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:27.551 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:27.552 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:27.570 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:27.571 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:27.571 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:27.571 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:27.571 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:27.571 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:27.571 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:27.571 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:27.572 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:27.590 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:27.591 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:27.591 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:27.591 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:27.591 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:27.591 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:27.591 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:27.591 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:27.592 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:27.610 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:27.610 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:27.611 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:27.611 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:27.611 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:27.611 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:27.611 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:27.611 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:27.611 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:27.631 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:27.631 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:27.632 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:27.632 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:27.632 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:27.632 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:27.632 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:27.632 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:27.632 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:27.691 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:27.691 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:27.692 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:27.692 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:27.692 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:27.692 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:27.692 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:27.692 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:27.692 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:27.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:28:27.707823 3893 master_proxy_rpc.cc:177] Failed to determine new leader Master: Timed out: ConnectToClusterRpc(addrs: 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127, num_attempts: 97) passed its deadline: Not found: no leader found: ConnectToClusterRpc(addrs: 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127, num_attempts: 1)
20:28:27.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:28:27.711252 3893 meta_cache.cc:788] Timed out: LookupRpcById { tablet: 'f9ccfdb32e164e61bde68775baab7f9d', attempt: 2 } failed: LookupRpcById timed out after deadline expired: LookupRpcById { tablet: 'f9ccfdb32e164e61bde68775baab7f9d', attempt: 2 } passed its deadline: Timed out: ConnectToClusterRpc(addrs: 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127, num_attempts: 97) passed its deadline: Not found: no leader found: ConnectToClusterRpc(addrs: 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127, num_attempts: 1)
20:28:27.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:28:27.713441 3893 master_proxy_rpc.cc:203] Re-attempting LookupRpcById request to leader Master (127.3.46.62:46071)
20:28:27.790 [DEBUG - pool-40-thread-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=259), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [0ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [1ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [20ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [40ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [60ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [80ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [80ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [101ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [161ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, deferred=Deferred@286014226(state=PENDING, result=null, callback=(continuation of Deferred@1954096262 after retry RPC after error@794810280) -> (continuation of Deferred@1922401760 after retry RPC after error@45212683) -> (continuation of Deferred@1062317519 after retry RPC after error@1860357914) -> (continuation of Deferred@805760419 after retry RPC after error@1025168916) -> (continuation of Deferred@253180845 after retry RPC after error@2004205386) -> (continuation of Deferred@1889091570 after retry RPC after error@467201411) -> (continuation of Deferred@765748998 after retry RPC after error@2003894807), errback=(continuation of Deferred@1954096262 after retry RPC after error@794810280) -> (continuation of Deferred@1922401760 after retry RPC after error@45212683) -> (continuation of Deferred@1062317519 after retry RPC after error@1860357914) -> (continuation of Deferred@805760419 after retry RPC after error@1025168916) -> (continuation of Deferred@253180845 after retry RPC after error@2004205386) -> (continuation of Deferred@1889091570 after retry RPC after error@467201411) -> (continuation of Deferred@765748998 after retry RPC after error@2003894807)))
20:28:27.791 [WARN - pool-40-thread-1] (HashedWheelTimer.java:716) An exception was thrown by TimerTask.
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713)
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701)
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:787)
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501)
at java.base/java.lang.Thread.run(Thread.java:840)
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713)
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701)
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:787)
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501)
at java.base/java.lang.Thread.run(Thread.java:840)
20:28:27.811 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:27.811 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:27.811 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:27.811 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:27.811 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:27.811 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:27.811 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:27.811 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:27.812 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:27.830 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:27.831 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:27.831 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:27.831 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:27.831 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:27.831 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:27.831 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:27.831 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:27.832 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:27.850 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:27.850 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:27.851 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:27.851 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:27.851 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:27.851 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:27.851 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:27.851 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:27.852 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:27.871 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:27.871 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:27.871 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:27.872 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:27.872 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:27.872 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:27.872 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:27.872 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:27.872 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:27.891 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:27.891 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:27.891 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:27.891 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:27.891 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:27.891 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:27.892 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:27.892 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:27.892 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:27.910 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:27.911 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:27.911 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:27.911 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:27.911 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:27.911 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:27.911 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:27.911 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:27.912 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:27.950 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:27.950 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:27.951 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:27.951 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:27.951 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:27.951 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:27.951 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:27.951 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:27.951 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:28.050 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:28.051 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:28.051 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:28.051 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:28.051 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:28.051 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:28.051 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:28.051 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:28.052 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:28.052 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=241), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, deferred=Deferred@758217006(state=PENDING, result=null, callback=(continuation of Deferred@1513891691 after retry RPC after error@130586477) -> (continuation of Deferred@1154215835 after retry RPC after error@1653937701) -> (continuation of Deferred@1504277091 after retry RPC after error@446293444) -> (continuation of Deferred@20280891 after retry RPC after error@1264286726) -> (continuation of Deferred@1509111140 after retry RPC after error@202460251) -> (continuation of Deferred@650078028 after retry RPC after error@907318481) -> (continuation of Deferred@1988943714 after retry RPC after error@1886987900), errback=(continuation of Deferred@1513891691 after retry RPC after error@130586477) -> (continuation of Deferred@1154215835 after retry RPC after error@1653937701) -> (continuation of Deferred@1504277091 after retry RPC after error@446293444) -> (continuation of Deferred@20280891 after retry RPC after error@1264286726) -> (continuation of Deferred@1509111140 after retry RPC after error@202460251) -> (continuation of Deferred@650078028 after retry RPC after error@907318481) -> (continuation of Deferred@1988943714 after retry RPC after error@1886987900)))
20:28:28.053 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=241), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, deferred=Deferred@758217006(state=PENDING, result=null, callback=(continuation of Deferred@1513891691 after retry RPC after error@130586477) -> (continuation of Deferred@1154215835 after retry RPC after error@1653937701) -> (continuation of Deferred@1504277091 after retry RPC after error@446293444) -> (continuation of Deferred@20280891 after retry RPC after error@1264286726) -> (continuation of Deferred@1509111140 after retry RPC after error@202460251) -> (continuation of Deferred@650078028 after retry RPC after error@907318481) -> (continuation of Deferred@1988943714 after retry RPC after error@1886987900), errback=(continuation of Deferred@1513891691 after retry RPC after error@130586477) -> (continuation of Deferred@1154215835 after retry RPC after error@1653937701) -> (continuation of Deferred@1504277091 after retry RPC after error@446293444) -> (continuation of Deferred@20280891 after retry RPC after error@1264286726) -> (continuation of Deferred@1509111140 after retry RPC after error@202460251) -> (continuation of Deferred@650078028 after retry RPC after error@907318481) -> (continuation of Deferred@1988943714 after retry RPC after error@1886987900)))
20:28:28.070 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:28.071 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:28.071 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:28.071 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:28.071 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:28.071 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:28.071 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:28.071 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:28.072 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:28.090 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:28.090 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:28.091 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:28.091 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:28.091 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:28.091 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:28.091 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:28.091 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:28.091 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:28.111 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:28.111 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:28.111 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:28.112 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:28.112 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:28.112 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:28.112 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:28.112 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:28.112 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:28.131 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:28.131 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:28.131 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:28.131 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:28.131 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:28.131 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:28.131 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:28.131 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:28.132 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:28.150 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:28.151 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:28.151 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:28.151 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:28.151 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:28.151 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:28.151 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:28.151 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:28.152 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:28.190 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:28.191 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:28.191 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:28.191 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:28.191 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:28.191 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:28.191 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:28.191 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:28.192 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:28.230 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:28.231 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:28.231 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:28.231 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:28.231 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:28.231 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:28.231 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:28.231 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:28.232 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:28.310 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:28.311 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:28.311 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:28.311 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:28.311 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:28.311 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:28.311 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:28.311 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:28.312 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:28.312 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=241), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@918892490(state=PENDING, result=null, callback=(continuation of Deferred@24718298 after retry RPC after error@460710823) -> (continuation of Deferred@610642036 after retry RPC after error@994146381) -> (continuation of Deferred@1932436228 after retry RPC after error@1207049757) -> (continuation of Deferred@1314728095 after retry RPC after error@2129572361) -> (continuation of Deferred@2027817043 after retry RPC after error@797701027) -> (continuation of Deferred@137707293 after retry RPC after error@54569704) -> (continuation of Deferred@190101059 after retry RPC after error@2066893409), errback=(continuation of Deferred@24718298 after retry RPC after error@460710823) -> (continuation of Deferred@610642036 after retry RPC after error@994146381) -> (continuation of Deferred@1932436228 after retry RPC after error@1207049757) -> (continuation of Deferred@1314728095 after retry RPC after error@2129572361) -> (continuation of Deferred@2027817043 after retry RPC after error@797701027) -> (continuation of Deferred@137707293 after retry RPC after error@54569704) -> (continuation of Deferred@190101059 after retry RPC after error@2066893409)))
20:28:28.312 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=241), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@918892490(state=PENDING, result=null, callback=(continuation of Deferred@24718298 after retry RPC after error@460710823) -> (continuation of Deferred@610642036 after retry RPC after error@994146381) -> (continuation of Deferred@1932436228 after retry RPC after error@1207049757) -> (continuation of Deferred@1314728095 after retry RPC after error@2129572361) -> (continuation of Deferred@2027817043 after retry RPC after error@797701027) -> (continuation of Deferred@137707293 after retry RPC after error@54569704) -> (continuation of Deferred@190101059 after retry RPC after error@2066893409), errback=(continuation of Deferred@24718298 after retry RPC after error@460710823) -> (continuation of Deferred@610642036 after retry RPC after error@994146381) -> (continuation of Deferred@1932436228 after retry RPC after error@1207049757) -> (continuation of Deferred@1314728095 after retry RPC after error@2129572361) -> (continuation of Deferred@2027817043 after retry RPC after error@797701027) -> (continuation of Deferred@137707293 after retry RPC after error@54569704) -> (continuation of Deferred@190101059 after retry RPC after error@2066893409)))
20:28:28.330 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:28.331 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:28.331 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:28.331 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:28.331 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:28.331 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:28.331 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:28.331 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:28.332 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:28.350 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:28.350 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:28.351 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:28.351 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:28.351 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:28.351 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:28.351 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:28.351 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:28.352 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:28.371 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:28.371 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:28.372 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:28.372 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:28.372 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:28.372 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:28.372 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:28.372 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:28.372 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:28.374 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:28.374 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:28.375 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:28.391 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:28.391 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:28.391 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:28.391 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:28.392 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:28.392 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:28.392 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:28.392 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:28.392 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:28.411 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:28.411 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:28.411 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:28.411 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:28.411 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:28.411 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:28.411 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:28.411 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:28.412 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:28.430 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:28.431 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:28.431 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:28.431 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:28.431 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:28.431 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:28.431 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:28.431 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:28.432 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:28.450 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:28.450 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:28.451 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:28.451 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:28.451 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:28.452 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:28.452 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:28.452 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:28.452 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:28.490 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:28.490 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:28.491 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:28.491 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:28.491 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:28.491 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:28.491 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:28.491 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:28.492 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:28.530 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:28.530 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:28.531 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:28.531 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:28.531 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:28.531 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:28.531 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:28.531 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:28.532 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:28.532 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=201), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [45ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [45ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@988676096(state=PENDING, result=null, callback=(continuation of Deferred@930909014 after retry RPC after error@194650283) -> (continuation of Deferred@394760028 after retry RPC after error@1562604460) -> (continuation of Deferred@248935781 after retry RPC after error@1410569068) -> (continuation of Deferred@2135231453 after retry RPC after error@960346506) -> (continuation of Deferred@83730069 after retry RPC after error@1287517589) -> (continuation of Deferred@460391484 after retry RPC after error@1330908977) -> (continuation of Deferred@657799544 after retry RPC after error@75985238) -> (continuation of Deferred@398775582 after retry RPC after error@2013687120), errback=(continuation of Deferred@930909014 after retry RPC after error@194650283) -> (continuation of Deferred@394760028 after retry RPC after error@1562604460) -> (continuation of Deferred@248935781 after retry RPC after error@1410569068) -> (continuation of Deferred@2135231453 after retry RPC after error@960346506) -> (continuation of Deferred@83730069 after retry RPC after error@1287517589) -> (continuation of Deferred@460391484 after retry RPC after error@1330908977) -> (continuation of Deferred@657799544 after retry RPC after error@75985238) -> (continuation of Deferred@398775582 after retry RPC after error@2013687120)))
20:28:28.533 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=201), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [45ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [45ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@988676096(state=PENDING, result=null, callback=(continuation of Deferred@930909014 after retry RPC after error@194650283) -> (continuation of Deferred@394760028 after retry RPC after error@1562604460) -> (continuation of Deferred@248935781 after retry RPC after error@1410569068) -> (continuation of Deferred@2135231453 after retry RPC after error@960346506) -> (continuation of Deferred@83730069 after retry RPC after error@1287517589) -> (continuation of Deferred@460391484 after retry RPC after error@1330908977) -> (continuation of Deferred@657799544 after retry RPC after error@75985238) -> (continuation of Deferred@398775582 after retry RPC after error@2013687120), errback=(continuation of Deferred@930909014 after retry RPC after error@194650283) -> (continuation of Deferred@394760028 after retry RPC after error@1562604460) -> (continuation of Deferred@248935781 after retry RPC after error@1410569068) -> (continuation of Deferred@2135231453 after retry RPC after error@960346506) -> (continuation of Deferred@83730069 after retry RPC after error@1287517589) -> (continuation of Deferred@460391484 after retry RPC after error@1330908977) -> (continuation of Deferred@657799544 after retry RPC after error@75985238) -> (continuation of Deferred@398775582 after retry RPC after error@2013687120)))
20:28:28.551 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:28.551 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:28.552 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:28.552 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:28.552 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:28.552 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:28.552 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:28.552 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:28.553 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:28.571 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:28.571 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:28.572 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:28.572 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:28.572 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:28.572 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:28.572 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:28.572 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:28.573 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:28.591 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:28.591 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:28.591 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:28.591 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:28.591 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:28.591 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:28.592 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:28.592 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:28.592 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:28.611 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:28.611 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:28.611 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:28.611 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:28.611 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:28.611 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:28.611 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:28.612 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:28.612 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:28.630 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:28.631 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:28.631 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:28.631 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:28.631 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:28.631 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:28.631 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:28.631 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:28.632 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:28.670 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:28.671 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:28.671 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:28.671 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:28.671 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:28.672 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:28.672 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:28.672 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:28.672 [WARN - kudu-nio-4] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:28.690 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:28.691 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:28.691 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:28.691 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:28.691 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:28.691 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:28.691 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:28.691 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:28.692 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:28.771 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:28.771 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:28.771 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:28.771 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:28.771 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:28.772 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:28.772 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:28.772 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:28.772 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:28.772 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=221), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [40ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [60ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [80ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [119ms] refreshing cache from master, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [120ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [121ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [139ms] refreshing cache from master, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [140ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [140ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [141ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [219ms] refreshing cache from master, [219ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [220ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [221ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@1982057070(state=PENDING, result=null, callback=(continuation of Deferred@439835605 after retry RPC after error@765805239) -> (continuation of Deferred@1838342647 after retry RPC after error@84555409) -> (continuation of Deferred@633547798 after retry RPC after error@586441443) -> (continuation of Deferred@1477685779 after retry RPC after error@217688382) -> (continuation of Deferred@812025643 after retry RPC after error@830736574) -> (continuation of Deferred@1950529463 after retry RPC after error@900939864) -> (continuation of Deferred@828954138 after retry RPC after error@540600689), errback=(continuation of Deferred@439835605 after retry RPC after error@765805239) -> (continuation of Deferred@1838342647 after retry RPC after error@84555409) -> (continuation of Deferred@633547798 after retry RPC after error@586441443) -> (continuation of Deferred@1477685779 after retry RPC after error@217688382) -> (continuation of Deferred@812025643 after retry RPC after error@830736574) -> (continuation of Deferred@1950529463 after retry RPC after error@900939864) -> (continuation of Deferred@828954138 after retry RPC after error@540600689)))
20:28:28.773 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=221), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [40ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [60ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [80ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [119ms] refreshing cache from master, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [120ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [121ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [139ms] refreshing cache from master, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [140ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [140ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [141ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [219ms] refreshing cache from master, [219ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [220ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [221ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@1982057070(state=PENDING, result=null, callback=(continuation of Deferred@439835605 after retry RPC after error@765805239) -> (continuation of Deferred@1838342647 after retry RPC after error@84555409) -> (continuation of Deferred@633547798 after retry RPC after error@586441443) -> (continuation of Deferred@1477685779 after retry RPC after error@217688382) -> (continuation of Deferred@812025643 after retry RPC after error@830736574) -> (continuation of Deferred@1950529463 after retry RPC after error@900939864) -> (continuation of Deferred@828954138 after retry RPC after error@540600689), errback=(continuation of Deferred@439835605 after retry RPC after error@765805239) -> (continuation of Deferred@1838342647 after retry RPC after error@84555409) -> (continuation of Deferred@633547798 after retry RPC after error@586441443) -> (continuation of Deferred@1477685779 after retry RPC after error@217688382) -> (continuation of Deferred@812025643 after retry RPC after error@830736574) -> (continuation of Deferred@1950529463 after retry RPC after error@900939864) -> (continuation of Deferred@828954138 after retry RPC after error@540600689)))
20:28:28.791 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:28.791 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:28.791 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:28.791 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:28.791 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:28.791 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:28.791 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:28.791 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:28.792 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:28.810 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:28.811 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:28.811 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:28.811 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:28.811 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:28.811 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:28.811 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:28.811 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:28.812 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:28.830 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:28.830 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:28.831 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:28.831 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:28.831 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:28.831 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:28.831 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:28.831 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:28.832 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:28.851 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:28.851 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:28.852 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:28.852 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:28.852 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:28.852 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:28.852 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:28.852 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:28.852 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:28.871 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:28.871 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:28.871 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:28.871 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:28.871 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:28.871 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:28.871 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:28.871 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:28.872 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:28.890 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:28.891 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:28.891 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:28.891 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:28.891 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:28.891 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:28.891 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:28.891 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:28.892 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:28.930 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:28.931 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:28.931 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:28.931 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:28.931 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:28.931 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:28.931 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:28.931 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:28.932 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:28.932 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=141), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@420997113(state=PENDING, result=null, callback=(continuation of Deferred@2114189003 after retry RPC after error@255511736) -> (continuation of Deferred@546074825 after retry RPC after error@100819726) -> (continuation of Deferred@1742108843 after retry RPC after error@1045440485) -> (continuation of Deferred@2107179742 after retry RPC after error@1754662000) -> (continuation of Deferred@1011956457 after retry RPC after error@50417033) -> (continuation of Deferred@909844677 after retry RPC after error@1851135990), errback=(continuation of Deferred@2114189003 after retry RPC after error@255511736) -> (continuation of Deferred@546074825 after retry RPC after error@100819726) -> (continuation of Deferred@1742108843 after retry RPC after error@1045440485) -> (continuation of Deferred@2107179742 after retry RPC after error@1754662000) -> (continuation of Deferred@1011956457 after retry RPC after error@50417033) -> (continuation of Deferred@909844677 after retry RPC after error@1851135990)))
20:28:28.933 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=141), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@420997113(state=PENDING, result=null, callback=(continuation of Deferred@2114189003 after retry RPC after error@255511736) -> (continuation of Deferred@546074825 after retry RPC after error@100819726) -> (continuation of Deferred@1742108843 after retry RPC after error@1045440485) -> (continuation of Deferred@2107179742 after retry RPC after error@1754662000) -> (continuation of Deferred@1011956457 after retry RPC after error@50417033) -> (continuation of Deferred@909844677 after retry RPC after error@1851135990), errback=(continuation of Deferred@2114189003 after retry RPC after error@255511736) -> (continuation of Deferred@546074825 after retry RPC after error@100819726) -> (continuation of Deferred@1742108843 after retry RPC after error@1045440485) -> (continuation of Deferred@2107179742 after retry RPC after error@1754662000) -> (continuation of Deferred@1011956457 after retry RPC after error@50417033) -> (continuation of Deferred@909844677 after retry RPC after error@1851135990)))
20:28:28.950 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:28.951 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:28.951 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:28.951 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:28.951 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:28.951 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:28.951 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:28.951 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:28.952 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:28.970 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:28.970 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:28.971 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:28.971 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:28.971 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:28.971 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:28.971 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:28.971 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:28.972 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:28.991 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:28.991 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:28.992 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:28.992 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:28.992 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:28.992 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:28.992 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:28.992 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:28.993 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:29.011 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:29.011 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:29.011 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:29.011 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:29.011 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:29.012 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:29.012 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:29.012 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:29.012 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:29.031 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:29.031 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:29.031 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:29.031 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:29.031 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:29.031 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:29.031 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:29.031 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:29.032 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:29.070 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:29.071 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:29.071 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:29.071 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:29.071 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:29.071 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:29.071 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:29.071 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:29.072 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:29.110 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:29.111 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:29.111 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:29.111 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:29.111 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:29.111 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:29.111 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:29.111 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:29.112 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:29.190 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:29.191 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:29.191 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:29.191 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:29.191 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:29.191 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:29.191 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:29.191 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:29.192 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:29.210 [DEBUG - pool-40-thread-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=259), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [43ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [60ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@1531761809(state=PENDING, result=null, callback=(continuation of Deferred@1344088929 after retry RPC after error@1251382304) -> (continuation of Deferred@1726156021 after retry RPC after error@1530311908) -> (continuation of Deferred@1657395904 after retry RPC after error@469787949) -> (continuation of Deferred@1660569340 after retry RPC after error@210063315) -> (continuation of Deferred@540449625 after retry RPC after error@1057012607) -> (continuation of Deferred@259905477 after retry RPC after error@690022958) -> (continuation of Deferred@1029197824 after retry RPC after error@129621911) -> (continuation of Deferred@1713107802 after retry RPC after error@1212417069), errback=(continuation of Deferred@1344088929 after retry RPC after error@1251382304) -> (continuation of Deferred@1726156021 after retry RPC after error@1530311908) -> (continuation of Deferred@1657395904 after retry RPC after error@469787949) -> (continuation of Deferred@1660569340 after retry RPC after error@210063315) -> (continuation of Deferred@540449625 after retry RPC after error@1057012607) -> (continuation of Deferred@259905477 after retry RPC after error@690022958) -> (continuation of Deferred@1029197824 after retry RPC after error@129621911) -> (continuation of Deferred@1713107802 after retry RPC after error@1212417069)))
20:28:29.211 [WARN - pool-40-thread-1] (HashedWheelTimer.java:716) An exception was thrown by TimerTask.
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713)
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701)
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:787)
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501)
at java.base/java.lang.Thread.run(Thread.java:840)
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713)
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701)
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:787)
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501)
at java.base/java.lang.Thread.run(Thread.java:840)
20:28:29.231 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:29.231 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:29.232 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:29.232 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:29.232 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:29.232 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:29.232 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:29.232 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:29.233 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:29.251 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:29.251 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:29.251 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:29.251 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:29.251 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:29.252 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:29.252 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:29.252 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:29.252 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:29.271 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:29.271 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:29.271 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:29.271 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:29.271 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:29.271 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:29.271 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:29.271 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:29.272 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:29.290 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:29.291 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:29.291 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:29.291 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:29.291 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:29.291 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:29.291 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:29.291 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:29.292 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:29.310 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:29.310 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:29.311 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:29.311 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:29.311 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:29.311 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:29.311 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:29.311 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:29.312 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:29.351 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:29.351 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:29.352 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:29.352 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:29.352 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:29.352 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:29.352 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:29.352 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:29.352 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:29.391 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:29.391 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:29.392 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:29.392 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:29.392 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:29.392 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:29.392 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:29.392 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:29.392 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:29.470 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:29.470 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:29.471 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:29.471 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:29.471 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:29.471 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:29.471 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:29.471 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:29.472 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:29.472 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=240), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [20ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [40ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [60ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [80ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [121ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [161ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [239ms] refreshing cache from master, [239ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [239ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [239ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [240ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [240ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [240ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [241ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@1001807120(state=PENDING, result=null, callback=(continuation of Deferred@1493966789 after retry RPC after error@850267799) -> (continuation of Deferred@1465938890 after retry RPC after error@1580727019) -> (continuation of Deferred@1744581855 after retry RPC after error@635720705) -> (continuation of Deferred@1900977492 after retry RPC after error@273662260) -> (continuation of Deferred@378754350 after retry RPC after error@290480033) -> (continuation of Deferred@343000290 after retry RPC after error@2052714809) -> (continuation of Deferred@1987047801 after retry RPC after error@1699812674), errback=(continuation of Deferred@1493966789 after retry RPC after error@850267799) -> (continuation of Deferred@1465938890 after retry RPC after error@1580727019) -> (continuation of Deferred@1744581855 after retry RPC after error@635720705) -> (continuation of Deferred@1900977492 after retry RPC after error@273662260) -> (continuation of Deferred@378754350 after retry RPC after error@290480033) -> (continuation of Deferred@343000290 after retry RPC after error@2052714809) -> (continuation of Deferred@1987047801 after retry RPC after error@1699812674)))
20:28:29.472 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=240), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [20ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [40ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [60ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [80ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [121ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [161ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [239ms] refreshing cache from master, [239ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [239ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [239ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [240ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [240ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [240ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [241ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@1001807120(state=PENDING, result=null, callback=(continuation of Deferred@1493966789 after retry RPC after error@850267799) -> (continuation of Deferred@1465938890 after retry RPC after error@1580727019) -> (continuation of Deferred@1744581855 after retry RPC after error@635720705) -> (continuation of Deferred@1900977492 after retry RPC after error@273662260) -> (continuation of Deferred@378754350 after retry RPC after error@290480033) -> (continuation of Deferred@343000290 after retry RPC after error@2052714809) -> (continuation of Deferred@1987047801 after retry RPC after error@1699812674), errback=(continuation of Deferred@1493966789 after retry RPC after error@850267799) -> (continuation of Deferred@1465938890 after retry RPC after error@1580727019) -> (continuation of Deferred@1744581855 after retry RPC after error@635720705) -> (continuation of Deferred@1900977492 after retry RPC after error@273662260) -> (continuation of Deferred@378754350 after retry RPC after error@290480033) -> (continuation of Deferred@343000290 after retry RPC after error@2052714809) -> (continuation of Deferred@1987047801 after retry RPC after error@1699812674)))
20:28:29.491 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:29.491 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:29.491 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:29.491 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:29.492 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:29.492 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:29.492 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:29.492 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:29.492 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:29.510 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:29.511 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:29.511 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:29.511 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:29.511 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:29.511 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:29.511 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:29.511 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:29.512 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:29.530 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:29.531 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:29.531 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:29.531 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:29.531 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:29.531 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:29.531 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:29.531 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:29.532 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:29.550 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:29.550 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:29.551 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:29.551 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:29.551 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:29.551 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:29.551 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:29.551 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:29.552 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:29.571 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:29.571 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:29.571 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:29.571 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:29.572 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:29.572 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:29.572 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:29.572 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:29.572 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:29.591 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:29.591 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:29.591 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:29.591 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:29.591 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:29.591 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:29.591 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:29.591 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:29.592 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:29.610 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:29.611 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:29.611 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:29.611 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:29.611 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:29.611 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:29.611 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:29.611 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:29.612 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:29.691 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:29.691 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:29.691 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:29.691 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:29.691 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:29.691 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:29.692 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:29.692 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:29.692 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:29.692 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=201), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [1ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [20ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [40ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [60ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [99ms] refreshing cache from master, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [100ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [100ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [101ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [119ms] refreshing cache from master, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [120ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [120ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [121ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [199ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [200ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [201ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@1526272626(state=PENDING, result=null, callback=(continuation of Deferred@1849038285 after retry RPC after error@1289005579) -> (continuation of Deferred@595268224 after retry RPC after error@830733855) -> (continuation of Deferred@1875185055 after retry RPC after error@111703710) -> (continuation of Deferred@2109512665 after retry RPC after error@1398923639) -> (continuation of Deferred@344958833 after retry RPC after error@1364602872) -> (continuation of Deferred@1219302488 after retry RPC after error@337329763) -> (continuation of Deferred@2101782462 after retry RPC after error@495943169), errback=(continuation of Deferred@1849038285 after retry RPC after error@1289005579) -> (continuation of Deferred@595268224 after retry RPC after error@830733855) -> (continuation of Deferred@1875185055 after retry RPC after error@111703710) -> (continuation of Deferred@2109512665 after retry RPC after error@1398923639) -> (continuation of Deferred@344958833 after retry RPC after error@1364602872) -> (continuation of Deferred@1219302488 after retry RPC after error@337329763) -> (continuation of Deferred@2101782462 after retry RPC after error@495943169)))
20:28:29.693 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=201), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [1ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [20ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [40ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [60ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [99ms] refreshing cache from master, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [100ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [100ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [101ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [119ms] refreshing cache from master, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [120ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [120ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [121ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [199ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [200ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [201ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@1526272626(state=PENDING, result=null, callback=(continuation of Deferred@1849038285 after retry RPC after error@1289005579) -> (continuation of Deferred@595268224 after retry RPC after error@830733855) -> (continuation of Deferred@1875185055 after retry RPC after error@111703710) -> (continuation of Deferred@2109512665 after retry RPC after error@1398923639) -> (continuation of Deferred@344958833 after retry RPC after error@1364602872) -> (continuation of Deferred@1219302488 after retry RPC after error@337329763) -> (continuation of Deferred@2101782462 after retry RPC after error@495943169), errback=(continuation of Deferred@1849038285 after retry RPC after error@1289005579) -> (continuation of Deferred@595268224 after retry RPC after error@830733855) -> (continuation of Deferred@1875185055 after retry RPC after error@111703710) -> (continuation of Deferred@2109512665 after retry RPC after error@1398923639) -> (continuation of Deferred@344958833 after retry RPC after error@1364602872) -> (continuation of Deferred@1219302488 after retry RPC after error@337329763) -> (continuation of Deferred@2101782462 after retry RPC after error@495943169)))
20:28:29.711 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:29.711 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:29.711 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:29.711 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:29.711 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:29.711 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:29.712 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:29.712 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:29.712 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:29.731 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:29.731 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:29.731 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:29.731 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:29.731 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:29.731 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:29.731 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:29.731 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:29.732 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:29.751 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:29.751 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:29.751 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:29.751 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:29.751 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:29.752 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:29.752 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:29.752 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:29.752 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:29.771 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:29.771 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:29.771 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:29.771 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:29.771 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:29.771 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:29.772 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:29.772 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:29.772 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:29.790 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:29.791 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:29.791 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:29.791 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:29.791 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:29.791 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:29.791 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:29.791 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:29.792 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:29.810 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:29.811 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:29.811 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:29.811 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:29.811 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:29.811 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:29.811 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:29.811 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:29.812 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:29.870 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:29.871 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:29.871 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:29.871 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:29.871 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:29.871 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:29.871 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:29.871 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:29.872 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:29.930 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:29.931 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:29.931 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:29.932 [INFO - pool-40-thread-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:29.932 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:29.932 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:29.932 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:29.932 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:29.932 [WARN - kudu-nio-6] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:29.950 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:29.950 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:29.951 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:29.951 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:29.951 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:29.951 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:29.951 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:29.951 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:29.952 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:29.952 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=241), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [222ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [222ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [222ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [222ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@938723853(state=PENDING, result=null, callback=(continuation of Deferred@333615693 after retry RPC after error@1171072532) -> (continuation of Deferred@1260260402 after retry RPC after error@1645483262) -> (continuation of Deferred@1709412875 after retry RPC after error@1236464933) -> (continuation of Deferred@1928732605 after retry RPC after error@814171897) -> (continuation of Deferred@1225648424 after retry RPC after error@398993137) -> (continuation of Deferred@1426180162 after retry RPC after error@1597766805) -> (continuation of Deferred@2035878643 after retry RPC after error@795886727) -> (continuation of Deferred@1953028851 after retry RPC after error@1958174655), errback=(continuation of Deferred@333615693 after retry RPC after error@1171072532) -> (continuation of Deferred@1260260402 after retry RPC after error@1645483262) -> (continuation of Deferred@1709412875 after retry RPC after error@1236464933) -> (continuation of Deferred@1928732605 after retry RPC after error@814171897) -> (continuation of Deferred@1225648424 after retry RPC after error@398993137) -> (continuation of Deferred@1426180162 after retry RPC after error@1597766805) -> (continuation of Deferred@2035878643 after retry RPC after error@795886727) -> (continuation of Deferred@1953028851 after retry RPC after error@1958174655)))
20:28:29.952 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=241), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [222ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [222ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [222ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [222ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@938723853(state=PENDING, result=null, callback=(continuation of Deferred@333615693 after retry RPC after error@1171072532) -> (continuation of Deferred@1260260402 after retry RPC after error@1645483262) -> (continuation of Deferred@1709412875 after retry RPC after error@1236464933) -> (continuation of Deferred@1928732605 after retry RPC after error@814171897) -> (continuation of Deferred@1225648424 after retry RPC after error@398993137) -> (continuation of Deferred@1426180162 after retry RPC after error@1597766805) -> (continuation of Deferred@2035878643 after retry RPC after error@795886727) -> (continuation of Deferred@1953028851 after retry RPC after error@1958174655), errback=(continuation of Deferred@333615693 after retry RPC after error@1171072532) -> (continuation of Deferred@1260260402 after retry RPC after error@1645483262) -> (continuation of Deferred@1709412875 after retry RPC after error@1236464933) -> (continuation of Deferred@1928732605 after retry RPC after error@814171897) -> (continuation of Deferred@1225648424 after retry RPC after error@398993137) -> (continuation of Deferred@1426180162 after retry RPC after error@1597766805) -> (continuation of Deferred@2035878643 after retry RPC after error@795886727) -> (continuation of Deferred@1953028851 after retry RPC after error@1958174655)))
20:28:29.971 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:29.971 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:29.972 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:29.972 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:29.972 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:29.972 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:29.972 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:29.972 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:29.973 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:29.991 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:29.991 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:29.991 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:29.991 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:29.991 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:29.991 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:29.991 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:29.991 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:29.992 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:30.010 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:30.011 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:30.011 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:30.011 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:30.011 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:30.011 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:30.011 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:30.011 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:30.013 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:30.030 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:30.030 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:30.031 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:30.031 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:30.031 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:30.031 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:30.031 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:30.031 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:30.032 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:30.051 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:30.051 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:30.052 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:30.052 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:30.052 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:30.052 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:30.052 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:30.053 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:30.053 [WARN - kudu-nio-2] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:30.071 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:30.071 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:30.071 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:30.071 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:30.071 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:30.071 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:30.072 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:30.072 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:30.072 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:30.090 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:30.091 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:30.091 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:30.091 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:30.091 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:30.091 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:30.091 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:30.091 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:30.092 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:30.211 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:30.211 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:30.211 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:30.212 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:30.212 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:30.212 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:30.212 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:30.212 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:30.212 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:30.213 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=241), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [20ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [40ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [60ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [99ms] refreshing cache from master, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [100ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [101ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [119ms] refreshing cache from master, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [120ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [120ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [121ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [241ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@395941552(state=PENDING, result=null, callback=(continuation of Deferred@1894215868 after retry RPC after error@1246497528) -> (continuation of Deferred@423137120 after retry RPC after error@1120904929) -> (continuation of Deferred@166091924 after retry RPC after error@601328794) -> (continuation of Deferred@2127119375 after retry RPC after error@1091444121) -> (continuation of Deferred@2050485200 after retry RPC after error@1696271184) -> (continuation of Deferred@1371980480 after retry RPC after error@1612050356) -> (continuation of Deferred@632942861 after retry RPC after error@415580001), errback=(continuation of Deferred@1894215868 after retry RPC after error@1246497528) -> (continuation of Deferred@423137120 after retry RPC after error@1120904929) -> (continuation of Deferred@166091924 after retry RPC after error@601328794) -> (continuation of Deferred@2127119375 after retry RPC after error@1091444121) -> (continuation of Deferred@2050485200 after retry RPC after error@1696271184) -> (continuation of Deferred@1371980480 after retry RPC after error@1612050356) -> (continuation of Deferred@632942861 after retry RPC after error@415580001)))
20:28:30.213 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=241), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [20ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [40ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [60ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [99ms] refreshing cache from master, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [100ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [101ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [119ms] refreshing cache from master, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [120ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [120ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [121ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [241ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@395941552(state=PENDING, result=null, callback=(continuation of Deferred@1894215868 after retry RPC after error@1246497528) -> (continuation of Deferred@423137120 after retry RPC after error@1120904929) -> (continuation of Deferred@166091924 after retry RPC after error@601328794) -> (continuation of Deferred@2127119375 after retry RPC after error@1091444121) -> (continuation of Deferred@2050485200 after retry RPC after error@1696271184) -> (continuation of Deferred@1371980480 after retry RPC after error@1612050356) -> (continuation of Deferred@632942861 after retry RPC after error@415580001), errback=(continuation of Deferred@1894215868 after retry RPC after error@1246497528) -> (continuation of Deferred@423137120 after retry RPC after error@1120904929) -> (continuation of Deferred@166091924 after retry RPC after error@601328794) -> (continuation of Deferred@2127119375 after retry RPC after error@1091444121) -> (continuation of Deferred@2050485200 after retry RPC after error@1696271184) -> (continuation of Deferred@1371980480 after retry RPC after error@1612050356) -> (continuation of Deferred@632942861 after retry RPC after error@415580001)))
20:28:30.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:28:30.215600 4132 master_runner.cc:186] Error getting master registration for 127.3.46.61:39407: Network error: Client connection negotiation failed: client connection to 127.3.46.61:39407: connect: Connection refused (error 111),
20:28:30.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:28:30.215767 4132 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
20:28:30.231 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:30.231 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:30.231 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:30.231 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:30.231 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:30.231 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:30.231 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:30.231 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:30.232 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:30.250 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:30.251 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:30.251 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:30.251 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:30.251 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:30.251 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:30.251 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:30.251 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:30.252 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:30.270 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:30.270 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:30.271 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:30.271 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:30.271 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:30.271 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:30.271 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:30.271 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:30.272 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:30.291 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:30.291 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:30.291 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:30.291 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:30.291 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:30.292 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:30.292 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:30.292 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:30.292 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:30.310 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:30.311 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:30.311 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:30.311 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:30.311 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:30.311 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:30.311 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:30.311 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:30.312 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:30.350 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:30.350 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:30.351 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:30.351 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:30.351 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:30.351 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:30.351 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:30.351 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:30.352 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:30.390 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:30.391 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:30.391 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:30.391 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:30.391 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:30.391 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:30.391 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:30.391 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:30.392 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:30.392 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=161), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@1510910192(state=PENDING, result=null, callback=(continuation of Deferred@1795271399 after retry RPC after error@99536906) -> (continuation of Deferred@495996975 after retry RPC after error@854995525) -> (continuation of Deferred@1422323822 after retry RPC after error@668669052) -> (continuation of Deferred@945105280 after retry RPC after error@1001537749) -> (continuation of Deferred@1486127693 after retry RPC after error@655558103) -> (continuation of Deferred@907747740 after retry RPC after error@520514402), errback=(continuation of Deferred@1795271399 after retry RPC after error@99536906) -> (continuation of Deferred@495996975 after retry RPC after error@854995525) -> (continuation of Deferred@1422323822 after retry RPC after error@668669052) -> (continuation of Deferred@945105280 after retry RPC after error@1001537749) -> (continuation of Deferred@1486127693 after retry RPC after error@655558103) -> (continuation of Deferred@907747740 after retry RPC after error@520514402)))
20:28:30.393 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=161), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@1510910192(state=PENDING, result=null, callback=(continuation of Deferred@1795271399 after retry RPC after error@99536906) -> (continuation of Deferred@495996975 after retry RPC after error@854995525) -> (continuation of Deferred@1422323822 after retry RPC after error@668669052) -> (continuation of Deferred@945105280 after retry RPC after error@1001537749) -> (continuation of Deferred@1486127693 after retry RPC after error@655558103) -> (continuation of Deferred@907747740 after retry RPC after error@520514402), errback=(continuation of Deferred@1795271399 after retry RPC after error@99536906) -> (continuation of Deferred@495996975 after retry RPC after error@854995525) -> (continuation of Deferred@1422323822 after retry RPC after error@668669052) -> (continuation of Deferred@945105280 after retry RPC after error@1001537749) -> (continuation of Deferred@1486127693 after retry RPC after error@655558103) -> (continuation of Deferred@907747740 after retry RPC after error@520514402)))
20:28:30.410 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:30.411 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:30.411 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:30.411 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:30.411 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:30.411 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:30.411 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:30.411 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:30.412 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:30.430 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:30.430 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:30.431 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:30.431 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:30.431 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:30.431 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:30.431 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:30.431 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:30.432 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:30.451 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:30.451 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:30.452 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:30.452 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:30.452 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:30.452 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:30.452 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:30.452 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:30.452 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:30.471 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:30.471 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:30.471 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:30.471 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:30.471 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:30.471 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:30.472 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:30.472 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:30.472 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:30.491 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:30.491 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:30.491 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:30.491 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:30.491 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:30.491 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:30.491 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:30.491 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:30.492 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:30.510 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:30.511 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:30.511 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:30.511 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:30.511 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:30.511 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:30.511 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:30.511 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:30.512 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:30.530 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:30.530 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:30.531 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:30.531 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:30.531 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:30.531 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:30.531 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:30.531 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:30.531 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:30.671 [DEBUG - pool-40-thread-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=260), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [121ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@634013927(state=PENDING, result=null, callback=(continuation of Deferred@328454470 after retry RPC after error@1237564128) -> (continuation of Deferred@663189999 after retry RPC after error@1358794888) -> (continuation of Deferred@1567187504 after retry RPC after error@153737159) -> (continuation of Deferred@368938764 after retry RPC after error@691545323) -> (continuation of Deferred@1071033118 after retry RPC after error@2022102220) -> (continuation of Deferred@33202495 after retry RPC after error@1770898071) -> (continuation of Deferred@1669929699 after retry RPC after error@761762538), errback=(continuation of Deferred@328454470 after retry RPC after error@1237564128) -> (continuation of Deferred@663189999 after retry RPC after error@1358794888) -> (continuation of Deferred@1567187504 after retry RPC after error@153737159) -> (continuation of Deferred@368938764 after retry RPC after error@691545323) -> (continuation of Deferred@1071033118 after retry RPC after error@2022102220) -> (continuation of Deferred@33202495 after retry RPC after error@1770898071) -> (continuation of Deferred@1669929699 after retry RPC after error@761762538)))
20:28:30.671 [WARN - pool-40-thread-1] (HashedWheelTimer.java:716) An exception was thrown by TimerTask.
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713)
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701)
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:787)
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501)
at java.base/java.lang.Thread.run(Thread.java:840)
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713)
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701)
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:787)
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501)
at java.base/java.lang.Thread.run(Thread.java:840)
20:28:30.691 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:30.691 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:30.692 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:30.692 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:30.692 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:30.692 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:30.692 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:30.692 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:30.693 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:30.711 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:30.711 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:30.711 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:30.711 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:30.712 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:30.712 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:30.712 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:30.712 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:30.712 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:30.731 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:30.731 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:30.731 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:30.731 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:30.731 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:30.731 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:30.731 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:30.731 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:30.732 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:30.750 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:30.751 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:30.751 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:30.751 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:30.751 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:30.751 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:30.751 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:30.751 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:30.752 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:30.770 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:30.770 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:30.771 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:30.771 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:30.771 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:30.771 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:30.771 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:30.771 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:30.772 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:30.811 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:30.811 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:30.812 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:30.812 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:30.812 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:30.812 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:30.812 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:30.812 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:30.812 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:30.870 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:30.871 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:30.871 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:30.871 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:30.871 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:30.871 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:30.871 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:30.871 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:30.872 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:30.951 [DEBUG - pool-40-thread-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=259), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [40ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [60ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [80ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [179ms] refreshing cache from master, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [180ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [180ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [181ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@263080120(state=PENDING, result=null, callback=(continuation of Deferred@268871449 after retry RPC after error@446402872) -> (continuation of Deferred@1748077079 after retry RPC after error@354939937) -> (continuation of Deferred@809281523 after retry RPC after error@1986585245) -> (continuation of Deferred@309182325 after retry RPC after error@952223717) -> (continuation of Deferred@1830288433 after retry RPC after error@1549440772) -> (continuation of Deferred@598937069 after retry RPC after error@360384657) -> (continuation of Deferred@1205835899 after retry RPC after error@1328928051), errback=(continuation of Deferred@268871449 after retry RPC after error@446402872) -> (continuation of Deferred@1748077079 after retry RPC after error@354939937) -> (continuation of Deferred@809281523 after retry RPC after error@1986585245) -> (continuation of Deferred@309182325 after retry RPC after error@952223717) -> (continuation of Deferred@1830288433 after retry RPC after error@1549440772) -> (continuation of Deferred@598937069 after retry RPC after error@360384657) -> (continuation of Deferred@1205835899 after retry RPC after error@1328928051)))
20:28:30.951 [WARN - pool-40-thread-1] (HashedWheelTimer.java:716) An exception was thrown by TimerTask.
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713)
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701)
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:787)
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501)
at java.base/java.lang.Thread.run(Thread.java:840)
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713)
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701)
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:787)
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501)
at java.base/java.lang.Thread.run(Thread.java:840)
20:28:30.971 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:30.971 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:30.971 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:30.971 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:30.971 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:30.971 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:30.971 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:30.971 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:30.972 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:30.990 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:30.991 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:30.991 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:30.991 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:30.991 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:30.991 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:30.991 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:30.991 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:30.992 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:31.010 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:31.010 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:31.011 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:31.011 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:31.011 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:31.011 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:31.011 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:31.011 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:31.012 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:31.030 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:31.030 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:31.031 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:31.031 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:31.031 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:31.031 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:31.031 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:31.031 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:31.032 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:31.051 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:31.051 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:31.051 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:31.051 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:31.051 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:31.051 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:31.052 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:31.052 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:31.052 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:31.071 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:31.071 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:31.072 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:31.072 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:31.072 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:31.072 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:31.072 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:31.072 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:31.073 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:31.130 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:31.130 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:31.131 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:31.131 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:31.131 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:31.131 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:31.131 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:31.131 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:31.131 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:31.151 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:31.151 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:31.151 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:31.151 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:31.151 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:31.152 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:31.152 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:31.152 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:31.152 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:31.153 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [103ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [161ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [181ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@1754095129(state=PENDING, result=null, callback=(continuation of Deferred@2061794215 after retry RPC after error@2104887630) -> (continuation of Deferred@269440489 after retry RPC after error@317766759) -> (continuation of Deferred@1267782877 after retry RPC after error@107055256) -> (continuation of Deferred@1823159712 after retry RPC after error@2075111382) -> (continuation of Deferred@919906882 after retry RPC after error@1023131976) -> (continuation of Deferred@164164930 after retry RPC after error@253704882) -> (continuation of Deferred@470419369 after retry RPC after error@2115593441), errback=(continuation of Deferred@2061794215 after retry RPC after error@2104887630) -> (continuation of Deferred@269440489 after retry RPC after error@317766759) -> (continuation of Deferred@1267782877 after retry RPC after error@107055256) -> (continuation of Deferred@1823159712 after retry RPC after error@2075111382) -> (continuation of Deferred@919906882 after retry RPC after error@1023131976) -> (continuation of Deferred@164164930 after retry RPC after error@253704882) -> (continuation of Deferred@470419369 after retry RPC after error@2115593441)))
20:28:31.153 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [103ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [161ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [181ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@1754095129(state=PENDING, result=null, callback=(continuation of Deferred@2061794215 after retry RPC after error@2104887630) -> (continuation of Deferred@269440489 after retry RPC after error@317766759) -> (continuation of Deferred@1267782877 after retry RPC after error@107055256) -> (continuation of Deferred@1823159712 after retry RPC after error@2075111382) -> (continuation of Deferred@919906882 after retry RPC after error@1023131976) -> (continuation of Deferred@164164930 after retry RPC after error@253704882) -> (continuation of Deferred@470419369 after retry RPC after error@2115593441), errback=(continuation of Deferred@2061794215 after retry RPC after error@2104887630) -> (continuation of Deferred@269440489 after retry RPC after error@317766759) -> (continuation of Deferred@1267782877 after retry RPC after error@107055256) -> (continuation of Deferred@1823159712 after retry RPC after error@2075111382) -> (continuation of Deferred@919906882 after retry RPC after error@1023131976) -> (continuation of Deferred@164164930 after retry RPC after error@253704882) -> (continuation of Deferred@470419369 after retry RPC after error@2115593441)))
20:28:31.171 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:31.171 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:31.171 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:31.171 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:31.171 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:31.171 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:31.171 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:31.171 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:31.172 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:31.190 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:31.191 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:31.191 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:31.191 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:31.191 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:31.191 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:31.191 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:31.191 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:31.192 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:31.210 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:31.210 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:31.211 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:31.211 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:31.211 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:31.211 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:31.211 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:31.211 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:31.212 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:31.231 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:31.231 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:31.232 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:31.232 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:31.232 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:31.232 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:31.232 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:31.232 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:31.232 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:31.251 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:31.251 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:31.251 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:31.251 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:31.251 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:31.251 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:31.251 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:31.251 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:31.252 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:31.291 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:31.291 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:31.291 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:31.291 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:31.292 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:31.292 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:31.292 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:31.292 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:31.292 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:31.311 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:31.311 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:31.311 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:31.311 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:31.311 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:31.311 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:31.311 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:31.311 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:31.312 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:31.330 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:31.331 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:31.331 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:31.331 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:31.331 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:31.331 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:31.331 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:31.331 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:31.332 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:31.410 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:31.411 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:31.411 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:31.411 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:31.411 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:31.411 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:31.411 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:31.411 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:31.412 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:31.412 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=241), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [120ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@548879297(state=PENDING, result=null, callback=(continuation of Deferred@710946846 after retry RPC after error@928680164) -> (continuation of Deferred@1167785377 after retry RPC after error@1550495712) -> (continuation of Deferred@1998689459 after retry RPC after error@350170907) -> (continuation of Deferred@104513492 after retry RPC after error@262538588) -> (continuation of Deferred@1952029719 after retry RPC after error@429118064) -> (continuation of Deferred@1250171529 after retry RPC after error@1974958688) -> (continuation of Deferred@1295448549 after retry RPC after error@1244770913), errback=(continuation of Deferred@710946846 after retry RPC after error@928680164) -> (continuation of Deferred@1167785377 after retry RPC after error@1550495712) -> (continuation of Deferred@1998689459 after retry RPC after error@350170907) -> (continuation of Deferred@104513492 after retry RPC after error@262538588) -> (continuation of Deferred@1952029719 after retry RPC after error@429118064) -> (continuation of Deferred@1250171529 after retry RPC after error@1974958688) -> (continuation of Deferred@1295448549 after retry RPC after error@1244770913)))
20:28:31.413 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=241), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [120ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@548879297(state=PENDING, result=null, callback=(continuation of Deferred@710946846 after retry RPC after error@928680164) -> (continuation of Deferred@1167785377 after retry RPC after error@1550495712) -> (continuation of Deferred@1998689459 after retry RPC after error@350170907) -> (continuation of Deferred@104513492 after retry RPC after error@262538588) -> (continuation of Deferred@1952029719 after retry RPC after error@429118064) -> (continuation of Deferred@1250171529 after retry RPC after error@1974958688) -> (continuation of Deferred@1295448549 after retry RPC after error@1244770913), errback=(continuation of Deferred@710946846 after retry RPC after error@928680164) -> (continuation of Deferred@1167785377 after retry RPC after error@1550495712) -> (continuation of Deferred@1998689459 after retry RPC after error@350170907) -> (continuation of Deferred@104513492 after retry RPC after error@262538588) -> (continuation of Deferred@1952029719 after retry RPC after error@429118064) -> (continuation of Deferred@1250171529 after retry RPC after error@1974958688) -> (continuation of Deferred@1295448549 after retry RPC after error@1244770913)))
20:28:31.430 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:31.431 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:31.431 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:31.431 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:31.431 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:31.431 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:31.431 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:31.431 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:31.432 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:31.450 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:31.450 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:31.451 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:31.451 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:31.451 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:31.451 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:31.451 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:31.451 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:31.452 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:31.470 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:31.470 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:31.471 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:31.471 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:31.471 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:31.471 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:31.471 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:31.471 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:31.471 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:31.491 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:31.491 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:31.491 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:31.491 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:31.491 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:31.492 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:31.492 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:31.492 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:31.492 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:31.511 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:31.511 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:31.511 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:31.511 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:31.511 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:31.511 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:31.511 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:31.511 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:31.512 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:31.530 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:31.531 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:31.531 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:31.531 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:31.531 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:31.531 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:31.531 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:31.531 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:31.532 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:31.570 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:31.570 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:31.571 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:31.571 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:31.571 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:31.571 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:31.571 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:31.571 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:31.571 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:31.650 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:31.651 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:31.651 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:31.651 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:31.651 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:31.651 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:31.651 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:31.651 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:31.652 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:31.652 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=221), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [222ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@788433797(state=PENDING, result=null, callback=(continuation of Deferred@1294150438 after retry RPC after error@1303449275) -> (continuation of Deferred@586939913 after retry RPC after error@2021911408) -> (continuation of Deferred@741180765 after retry RPC after error@1764524725) -> (continuation of Deferred@341455407 after retry RPC after error@1944421311) -> (continuation of Deferred@857925619 after retry RPC after error@1041687048) -> (continuation of Deferred@2065200303 after retry RPC after error@616560648) -> (continuation of Deferred@433683 after retry RPC after error@151595705), errback=(continuation of Deferred@1294150438 after retry RPC after error@1303449275) -> (continuation of Deferred@586939913 after retry RPC after error@2021911408) -> (continuation of Deferred@741180765 after retry RPC after error@1764524725) -> (continuation of Deferred@341455407 after retry RPC after error@1944421311) -> (continuation of Deferred@857925619 after retry RPC after error@1041687048) -> (continuation of Deferred@2065200303 after retry RPC after error@616560648) -> (continuation of Deferred@433683 after retry RPC after error@151595705)))
20:28:31.653 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=221), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [222ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@788433797(state=PENDING, result=null, callback=(continuation of Deferred@1294150438 after retry RPC after error@1303449275) -> (continuation of Deferred@586939913 after retry RPC after error@2021911408) -> (continuation of Deferred@741180765 after retry RPC after error@1764524725) -> (continuation of Deferred@341455407 after retry RPC after error@1944421311) -> (continuation of Deferred@857925619 after retry RPC after error@1041687048) -> (continuation of Deferred@2065200303 after retry RPC after error@616560648) -> (continuation of Deferred@433683 after retry RPC after error@151595705), errback=(continuation of Deferred@1294150438 after retry RPC after error@1303449275) -> (continuation of Deferred@586939913 after retry RPC after error@2021911408) -> (continuation of Deferred@741180765 after retry RPC after error@1764524725) -> (continuation of Deferred@341455407 after retry RPC after error@1944421311) -> (continuation of Deferred@857925619 after retry RPC after error@1041687048) -> (continuation of Deferred@2065200303 after retry RPC after error@616560648) -> (continuation of Deferred@433683 after retry RPC after error@151595705)))
20:28:31.670 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:31.670 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:31.671 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:31.671 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:31.671 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:31.671 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:31.671 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:31.671 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:31.672 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:31.691 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:31.691 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:31.692 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:31.692 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:31.692 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:31.692 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:31.692 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:31.692 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:31.692 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:31.711 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:31.711 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:31.711 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:31.711 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:31.711 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:31.712 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:31.712 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:31.712 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:31.712 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:31.730 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:31.731 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:31.731 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:31.731 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:31.731 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:31.731 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:31.731 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:31.731 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:31.732 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:31.750 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:31.750 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:31.751 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:31.751 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:31.751 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:31.751 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:31.751 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:31.751 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:31.752 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:31.791 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:31.791 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:31.792 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:31.792 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:31.792 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:31.792 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:31.792 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:31.792 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:31.792 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:31.811 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:31.811 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:31.811 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:31.811 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:31.811 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:31.812 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:31.812 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:31.812 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:31.812 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:31.871 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:31.871 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:31.872 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:31.872 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:31.872 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:31.872 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:31.872 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:31.872 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:31.873 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:31.873 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [141ms] refreshing cache from master, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [201ms] refreshing cache from master, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [203ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [203ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@1663301653(state=PENDING, result=null, callback=(continuation of Deferred@1760943981 after retry RPC after error@1177963628) -> (continuation of Deferred@2098008886 after retry RPC after error@26390482) -> (continuation of Deferred@1197086266 after retry RPC after error@872118161) -> (continuation of Deferred@645547643 after retry RPC after error@839029081) -> (continuation of Deferred@637446885 after retry RPC after error@1297155636) -> (continuation of Deferred@971823227 after retry RPC after error@532815430) -> (continuation of Deferred@1329280729 after retry RPC after error@178190764), errback=(continuation of Deferred@1760943981 after retry RPC after error@1177963628) -> (continuation of Deferred@2098008886 after retry RPC after error@26390482) -> (continuation of Deferred@1197086266 after retry RPC after error@872118161) -> (continuation of Deferred@645547643 after retry RPC after error@839029081) -> (continuation of Deferred@637446885 after retry RPC after error@1297155636) -> (continuation of Deferred@971823227 after retry RPC after error@532815430) -> (continuation of Deferred@1329280729 after retry RPC after error@178190764)))
20:28:31.873 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [141ms] refreshing cache from master, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [201ms] refreshing cache from master, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [203ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [203ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@1663301653(state=PENDING, result=null, callback=(continuation of Deferred@1760943981 after retry RPC after error@1177963628) -> (continuation of Deferred@2098008886 after retry RPC after error@26390482) -> (continuation of Deferred@1197086266 after retry RPC after error@872118161) -> (continuation of Deferred@645547643 after retry RPC after error@839029081) -> (continuation of Deferred@637446885 after retry RPC after error@1297155636) -> (continuation of Deferred@971823227 after retry RPC after error@532815430) -> (continuation of Deferred@1329280729 after retry RPC after error@178190764), errback=(continuation of Deferred@1760943981 after retry RPC after error@1177963628) -> (continuation of Deferred@2098008886 after retry RPC after error@26390482) -> (continuation of Deferred@1197086266 after retry RPC after error@872118161) -> (continuation of Deferred@645547643 after retry RPC after error@839029081) -> (continuation of Deferred@637446885 after retry RPC after error@1297155636) -> (continuation of Deferred@971823227 after retry RPC after error@532815430) -> (continuation of Deferred@1329280729 after retry RPC after error@178190764)))
20:28:31.891 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:31.891 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:31.891 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:31.891 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:31.892 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:31.892 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:31.892 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:31.892 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:31.892 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:31.910 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:31.911 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:31.911 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:31.911 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:31.911 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:31.911 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:31.911 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:31.911 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:31.912 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:31.930 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:31.930 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:31.931 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:31.931 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:31.931 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:31.931 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:31.931 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:31.931 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:31.932 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:31.951 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:31.951 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:31.952 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:31.952 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:31.952 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:31.952 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:31.952 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:31.952 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:31.952 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:31.971 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:31.971 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:31.971 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:31.971 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:31.972 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:31.972 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:31.972 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:31.972 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:31.972 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:31.991 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:31.991 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:31.991 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:31.991 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:31.991 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:31.991 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:31.991 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:31.991 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:31.992 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:32.010 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:32.011 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:32.011 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:32.011 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:32.011 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:32.011 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:32.011 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:32.011 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:32.012 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:32.151 [DEBUG - pool-40-thread-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=259), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [1ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [20ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [40ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [99ms] refreshing cache from master, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [100ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [100ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [101ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [119ms] refreshing cache from master, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [120ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [120ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [121ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@1222431127(state=PENDING, result=null, callback=(continuation of Deferred@226426193 after retry RPC after error@632874302) -> (continuation of Deferred@922224407 after retry RPC after error@403056694) -> (continuation of Deferred@1651034642 after retry RPC after error@1790580) -> (continuation of Deferred@1991060959 after retry RPC after error@1606679716) -> (continuation of Deferred@2069861652 after retry RPC after error@554561785) -> (continuation of Deferred@1053942482 after retry RPC after error@2125930352) -> (continuation of Deferred@2034750087 after retry RPC after error@931360622), errback=(continuation of Deferred@226426193 after retry RPC after error@632874302) -> (continuation of Deferred@922224407 after retry RPC after error@403056694) -> (continuation of Deferred@1651034642 after retry RPC after error@1790580) -> (continuation of Deferred@1991060959 after retry RPC after error@1606679716) -> (continuation of Deferred@2069861652 after retry RPC after error@554561785) -> (continuation of Deferred@1053942482 after retry RPC after error@2125930352) -> (continuation of Deferred@2034750087 after retry RPC after error@931360622)))
20:28:32.151 [WARN - pool-40-thread-1] (HashedWheelTimer.java:716) An exception was thrown by TimerTask.
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713)
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701)
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:787)
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501)
at java.base/java.lang.Thread.run(Thread.java:840)
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713)
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701)
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:787)
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501)
at java.base/java.lang.Thread.run(Thread.java:840)
20:28:32.171 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:32.171 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:32.172 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:32.172 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:32.172 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:32.172 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:32.172 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:32.172 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:32.172 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:32.191 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:32.191 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:32.191 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:32.191 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:32.191 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:32.191 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:32.191 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:32.191 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:32.192 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:32.210 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:32.211 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:32.211 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:32.211 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:32.211 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:32.211 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:32.211 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:32.211 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:32.212 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:32.230 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:32.231 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:32.231 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:32.232 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:32.232 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:32.232 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:32.232 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:32.232 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:32.232 [WARN - kudu-nio-2] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:32.250 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:32.250 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:32.251 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:32.251 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:32.251 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:32.251 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:32.251 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:32.251 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:32.252 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:32.271 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:32.271 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:32.272 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:32.272 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:32.272 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:32.272 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:32.272 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:32.272 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:32.273 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:32.291 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:32.291 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:32.291 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:32.291 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:32.291 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:32.291 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:32.291 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:32.291 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:32.292 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:32.310 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:32.311 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:32.311 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:32.311 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:32.311 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:32.311 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:32.311 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:32.311 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:32.312 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:32.391 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:32.391 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:32.391 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:32.391 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:32.391 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:32.391 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:32.391 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:32.392 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:32.392 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:32.392 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=221), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [1ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [20ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [40ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [61ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [80ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [119ms] refreshing cache from master, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [120ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [120ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [121ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [139ms] refreshing cache from master, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [140ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [140ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [141ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [219ms] refreshing cache from master, [219ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [219ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [220ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [220ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [221ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, deferred=Deferred@1900391411(state=PENDING, result=null, callback=(continuation of Deferred@1637320253 after retry RPC after error@1048658479) -> (continuation of Deferred@1639526596 after retry RPC after error@1040127283) -> (continuation of Deferred@1619319519 after retry RPC after error@1419799770) -> (continuation of Deferred@2083998106 after retry RPC after error@967085474) -> (continuation of Deferred@2133916267 after retry RPC after error@681680875) -> (continuation of Deferred@1855744252 after retry RPC after error@1934106896) -> (continuation of Deferred@198409904 after retry RPC after error@791279334) -> (continuation of Deferred@587178103 after retry RPC after error@269305241), errback=(continuation of Deferred@1637320253 after retry RPC after error@1048658479) -> (continuation of Deferred@1639526596 after retry RPC after error@1040127283) -> (continuation of Deferred@1619319519 after retry RPC after error@1419799770) -> (continuation of Deferred@2083998106 after retry RPC after error@967085474) -> (continuation of Deferred@2133916267 after retry RPC after error@681680875) -> (continuation of Deferred@1855744252 after retry RPC after error@1934106896) -> (continuation of Deferred@198409904 after retry RPC after error@791279334) -> (continuation of Deferred@587178103 after retry RPC after error@269305241)))
20:28:32.393 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=221), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [1ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [20ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [40ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [61ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [80ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [119ms] refreshing cache from master, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [120ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [120ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [121ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [139ms] refreshing cache from master, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [140ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [140ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [141ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [219ms] refreshing cache from master, [219ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [219ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [220ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [220ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [221ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, deferred=Deferred@1900391411(state=PENDING, result=null, callback=(continuation of Deferred@1637320253 after retry RPC after error@1048658479) -> (continuation of Deferred@1639526596 after retry RPC after error@1040127283) -> (continuation of Deferred@1619319519 after retry RPC after error@1419799770) -> (continuation of Deferred@2083998106 after retry RPC after error@967085474) -> (continuation of Deferred@2133916267 after retry RPC after error@681680875) -> (continuation of Deferred@1855744252 after retry RPC after error@1934106896) -> (continuation of Deferred@198409904 after retry RPC after error@791279334) -> (continuation of Deferred@587178103 after retry RPC after error@269305241), errback=(continuation of Deferred@1637320253 after retry RPC after error@1048658479) -> (continuation of Deferred@1639526596 after retry RPC after error@1040127283) -> (continuation of Deferred@1619319519 after retry RPC after error@1419799770) -> (continuation of Deferred@2083998106 after retry RPC after error@967085474) -> (continuation of Deferred@2133916267 after retry RPC after error@681680875) -> (continuation of Deferred@1855744252 after retry RPC after error@1934106896) -> (continuation of Deferred@198409904 after retry RPC after error@791279334) -> (continuation of Deferred@587178103 after retry RPC after error@269305241)))
20:28:32.410 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:32.411 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:32.411 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:32.411 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:32.411 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:32.411 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:32.411 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:32.411 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:32.412 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:32.430 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:32.430 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:32.431 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:32.431 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:32.431 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:32.431 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:32.431 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:32.431 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:32.432 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:32.451 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:32.451 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:32.451 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:32.451 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:32.451 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:32.451 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:32.452 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:32.452 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:32.453 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:32.470 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:32.471 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:32.471 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:32.471 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:32.471 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:32.471 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:32.471 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:32.471 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:32.473 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:32.490 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:32.491 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:32.491 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:32.491 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:32.491 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:32.491 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:32.491 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:32.491 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:32.492 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:32.530 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:32.531 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:32.531 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:32.531 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:32.531 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:32.531 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:32.531 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:32.532 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:32.532 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:32.590 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:32.591 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:32.591 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:32.591 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:32.591 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:32.591 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:32.591 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:32.591 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:32.592 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:32.611 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:32.611 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:32.611 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:32.611 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:32.611 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:32.611 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:32.611 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:32.611 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:32.613 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:32.613 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [43ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [203ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [203ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@421852704(state=PENDING, result=null, callback=(continuation of Deferred@1460687034 after retry RPC after error@1493454801) -> (continuation of Deferred@1890795131 after retry RPC after error@1318605755) -> (continuation of Deferred@467165010 after retry RPC after error@1624891711) -> (continuation of Deferred@158354361 after retry RPC after error@91870375) -> (continuation of Deferred@1552034197 after retry RPC after error@1946020643) -> (continuation of Deferred@2027641702 after retry RPC after error@860648502) -> (continuation of Deferred@2127506763 after retry RPC after error@1651991874), errback=(continuation of Deferred@1460687034 after retry RPC after error@1493454801) -> (continuation of Deferred@1890795131 after retry RPC after error@1318605755) -> (continuation of Deferred@467165010 after retry RPC after error@1624891711) -> (continuation of Deferred@158354361 after retry RPC after error@91870375) -> (continuation of Deferred@1552034197 after retry RPC after error@1946020643) -> (continuation of Deferred@2027641702 after retry RPC after error@860648502) -> (continuation of Deferred@2127506763 after retry RPC after error@1651991874)))
20:28:32.614 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [43ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [203ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [203ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@421852704(state=PENDING, result=null, callback=(continuation of Deferred@1460687034 after retry RPC after error@1493454801) -> (continuation of Deferred@1890795131 after retry RPC after error@1318605755) -> (continuation of Deferred@467165010 after retry RPC after error@1624891711) -> (continuation of Deferred@158354361 after retry RPC after error@91870375) -> (continuation of Deferred@1552034197 after retry RPC after error@1946020643) -> (continuation of Deferred@2027641702 after retry RPC after error@860648502) -> (continuation of Deferred@2127506763 after retry RPC after error@1651991874), errback=(continuation of Deferred@1460687034 after retry RPC after error@1493454801) -> (continuation of Deferred@1890795131 after retry RPC after error@1318605755) -> (continuation of Deferred@467165010 after retry RPC after error@1624891711) -> (continuation of Deferred@158354361 after retry RPC after error@91870375) -> (continuation of Deferred@1552034197 after retry RPC after error@1946020643) -> (continuation of Deferred@2027641702 after retry RPC after error@860648502) -> (continuation of Deferred@2127506763 after retry RPC after error@1651991874)))
20:28:32.630 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:32.631 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:32.631 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:32.631 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:32.631 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:32.631 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:32.631 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:32.631 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:32.632 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:32.650 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:32.650 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:32.651 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:32.651 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:32.651 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:32.651 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:32.651 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:32.651 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:32.652 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:32.671 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:32.671 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:32.671 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:32.671 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:32.671 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:32.672 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:32.672 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:32.672 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:32.672 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:32.691 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:32.691 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:32.691 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:32.691 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:32.691 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:32.691 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:32.691 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:32.691 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:32.692 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:32.710 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:32.711 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:32.711 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:32.711 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:32.711 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:32.711 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:32.711 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:32.711 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:32.712 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:32.750 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:32.750 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:32.751 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:32.751 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:32.751 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:32.751 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:32.751 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:32.751 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:32.752 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:32.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:28:32.766999 3893 master_proxy_rpc.cc:177] Failed to determine new leader Master: Timed out: ConnectToClusterRpc(addrs: 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127, num_attempts: 97) passed its deadline: Not found: no leader found: ConnectToClusterRpc(addrs: 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127, num_attempts: 1)
20:28:32.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:28:32.768378 3893 meta_cache.cc:788] Timed out: LookupRpcById { tablet: 'f9ccfdb32e164e61bde68775baab7f9d', attempt: 2 } failed: LookupRpcById timed out after deadline expired: LookupRpcById { tablet: 'f9ccfdb32e164e61bde68775baab7f9d', attempt: 2 } passed its deadline: Timed out: ConnectToClusterRpc(addrs: 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127, num_attempts: 97) passed its deadline: Not found: no leader found: ConnectToClusterRpc(addrs: 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127, num_attempts: 1)
20:28:32.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:28:32.770299 3893 master_proxy_rpc.cc:203] Re-attempting LookupRpcById request to leader Master (127.3.46.62:46071)
20:28:32.771 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:32.771 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:32.771 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:32.771 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:32.771 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:32.771 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:32.772 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:32.772 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:32.772 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:32.890 [DEBUG - pool-40-thread-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=259), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [141ms] refreshing cache from master, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [142ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@829196348(state=PENDING, result=null, callback=(continuation of Deferred@894787442 after retry RPC after error@309194684) -> (continuation of Deferred@329439605 after retry RPC after error@1180339797) -> (continuation of Deferred@1508632355 after retry RPC after error@612823981) -> (continuation of Deferred@560656185 after retry RPC after error@869332327) -> (continuation of Deferred@889511617 after retry RPC after error@638343255) -> (continuation of Deferred@1357473928 after retry RPC after error@1680262207) -> (continuation of Deferred@2009590956 after retry RPC after error@860736739), errback=(continuation of Deferred@894787442 after retry RPC after error@309194684) -> (continuation of Deferred@329439605 after retry RPC after error@1180339797) -> (continuation of Deferred@1508632355 after retry RPC after error@612823981) -> (continuation of Deferred@560656185 after retry RPC after error@869332327) -> (continuation of Deferred@889511617 after retry RPC after error@638343255) -> (continuation of Deferred@1357473928 after retry RPC after error@1680262207) -> (continuation of Deferred@2009590956 after retry RPC after error@860736739)))
20:28:32.891 [WARN - pool-40-thread-1] (HashedWheelTimer.java:716) An exception was thrown by TimerTask.
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713)
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701)
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:787)
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501)
at java.base/java.lang.Thread.run(Thread.java:840)
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713)
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701)
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:787)
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501)
at java.base/java.lang.Thread.run(Thread.java:840)
20:28:32.910 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:32.910 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:32.911 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:32.911 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:32.911 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:32.911 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:32.911 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:32.911 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:32.911 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:32.931 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:32.931 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:32.932 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:32.932 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:32.932 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:32.932 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:32.932 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:32.932 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:32.933 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:32.951 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:32.951 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:32.951 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:32.951 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:32.952 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:32.952 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:32.952 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:32.952 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:32.952 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:32.971 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:32.971 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:32.971 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:32.971 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:32.972 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:32.972 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:32.972 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:32.972 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:32.972 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:32.991 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:32.991 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:32.991 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:32.991 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:32.991 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:32.991 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:32.991 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:32.992 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:32.992 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:33.031 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:33.031 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:33.031 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:33.031 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:33.031 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:33.031 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:33.032 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:33.032 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:33.032 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:33.071 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:33.071 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:33.071 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:33.071 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:33.071 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:33.071 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:33.071 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:33.071 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:33.072 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:33.110 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:33.111 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:33.111 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:33.111 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:33.111 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:33.111 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:33.111 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:33.111 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:33.112 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:33.112 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [1ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [23ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@1382482972(state=PENDING, result=null, callback=(continuation of Deferred@567491625 after retry RPC after error@1904160877) -> (continuation of Deferred@1274575300 after retry RPC after error@559272771) -> (continuation of Deferred@434078004 after retry RPC after error@288040506) -> (continuation of Deferred@429214594 after retry RPC after error@374692920) -> (continuation of Deferred@1936112103 after retry RPC after error@993286983) -> (continuation of Deferred@147771128 after retry RPC after error@877061957) -> (continuation of Deferred@1923729835 after retry RPC after error@1809730266), errback=(continuation of Deferred@567491625 after retry RPC after error@1904160877) -> (continuation of Deferred@1274575300 after retry RPC after error@559272771) -> (continuation of Deferred@434078004 after retry RPC after error@288040506) -> (continuation of Deferred@429214594 after retry RPC after error@374692920) -> (continuation of Deferred@1936112103 after retry RPC after error@993286983) -> (continuation of Deferred@147771128 after retry RPC after error@877061957) -> (continuation of Deferred@1923729835 after retry RPC after error@1809730266)))
20:28:33.112 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [1ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [23ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@1382482972(state=PENDING, result=null, callback=(continuation of Deferred@567491625 after retry RPC after error@1904160877) -> (continuation of Deferred@1274575300 after retry RPC after error@559272771) -> (continuation of Deferred@434078004 after retry RPC after error@288040506) -> (continuation of Deferred@429214594 after retry RPC after error@374692920) -> (continuation of Deferred@1936112103 after retry RPC after error@993286983) -> (continuation of Deferred@147771128 after retry RPC after error@877061957) -> (continuation of Deferred@1923729835 after retry RPC after error@1809730266), errback=(continuation of Deferred@567491625 after retry RPC after error@1904160877) -> (continuation of Deferred@1274575300 after retry RPC after error@559272771) -> (continuation of Deferred@434078004 after retry RPC after error@288040506) -> (continuation of Deferred@429214594 after retry RPC after error@374692920) -> (continuation of Deferred@1936112103 after retry RPC after error@993286983) -> (continuation of Deferred@147771128 after retry RPC after error@877061957) -> (continuation of Deferred@1923729835 after retry RPC after error@1809730266)))
20:28:33.130 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:33.131 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:33.131 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:33.131 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:33.131 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:33.131 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:33.131 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:33.131 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:33.132 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:33.150 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:33.150 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:33.151 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:33.151 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:33.151 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:33.151 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:33.151 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:33.151 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:33.152 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:33.170 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:33.171 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:33.171 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:33.171 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:33.171 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:33.171 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:33.171 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:33.171 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:33.172 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:33.190 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:33.191 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:33.191 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:33.191 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:33.191 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:33.191 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:33.191 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:33.191 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:33.192 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:33.210 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:33.210 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:33.211 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:33.211 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:33.211 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:33.211 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:33.211 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:33.211 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:33.212 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:33.231 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:33.231 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:33.231 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:33.232 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:33.232 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:33.232 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:33.232 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:33.232 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:33.232 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:33.251 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:33.251 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:33.251 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:33.251 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:33.251 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:33.251 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:33.251 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:33.252 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:33.252 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:33.311 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:33.311 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:33.311 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:33.311 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:33.311 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:33.311 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:33.311 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:33.312 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:33.312 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:33.312 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [120ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [180ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@1302836503(state=PENDING, result=null, callback=(continuation of Deferred@376852557 after retry RPC after error@1553967301) -> (continuation of Deferred@1187817087 after retry RPC after error@917744828) -> (continuation of Deferred@316892703 after retry RPC after error@857419014) -> (continuation of Deferred@281516612 after retry RPC after error@1002050220) -> (continuation of Deferred@853563482 after retry RPC after error@1492721693) -> (continuation of Deferred@1950389500 after retry RPC after error@528795562) -> (continuation of Deferred@790003336 after retry RPC after error@1056852671), errback=(continuation of Deferred@376852557 after retry RPC after error@1553967301) -> (continuation of Deferred@1187817087 after retry RPC after error@917744828) -> (continuation of Deferred@316892703 after retry RPC after error@857419014) -> (continuation of Deferred@281516612 after retry RPC after error@1002050220) -> (continuation of Deferred@853563482 after retry RPC after error@1492721693) -> (continuation of Deferred@1950389500 after retry RPC after error@528795562) -> (continuation of Deferred@790003336 after retry RPC after error@1056852671)))
20:28:33.313 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [120ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [180ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@1302836503(state=PENDING, result=null, callback=(continuation of Deferred@376852557 after retry RPC after error@1553967301) -> (continuation of Deferred@1187817087 after retry RPC after error@917744828) -> (continuation of Deferred@316892703 after retry RPC after error@857419014) -> (continuation of Deferred@281516612 after retry RPC after error@1002050220) -> (continuation of Deferred@853563482 after retry RPC after error@1492721693) -> (continuation of Deferred@1950389500 after retry RPC after error@528795562) -> (continuation of Deferred@790003336 after retry RPC after error@1056852671), errback=(continuation of Deferred@376852557 after retry RPC after error@1553967301) -> (continuation of Deferred@1187817087 after retry RPC after error@917744828) -> (continuation of Deferred@316892703 after retry RPC after error@857419014) -> (continuation of Deferred@281516612 after retry RPC after error@1002050220) -> (continuation of Deferred@853563482 after retry RPC after error@1492721693) -> (continuation of Deferred@1950389500 after retry RPC after error@528795562) -> (continuation of Deferred@790003336 after retry RPC after error@1056852671)))
20:28:33.330 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:33.331 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:33.331 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:33.331 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:33.331 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:33.331 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:33.331 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:33.332 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:33.332 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:33.350 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:33.351 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:33.351 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:33.351 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:33.351 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:33.351 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:33.351 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:33.351 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:33.352 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:33.370 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:33.371 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:33.371 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:33.371 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:33.371 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:33.371 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:33.371 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:33.371 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:33.372 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:33.390 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:33.390 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:33.391 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:33.391 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:33.391 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:33.391 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:33.391 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:33.391 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:33.391 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:33.411 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:33.411 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:33.412 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:33.412 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:33.412 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:33.412 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:33.412 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:33.412 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:33.412 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:33.431 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:33.431 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:33.431 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:33.431 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:33.431 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:33.431 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:33.432 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:33.432 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:33.432 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:33.491 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:33.492 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:33.492 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:33.492 [INFO - pool-40-thread-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:33.492 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:33.492 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:33.493 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:33.493 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:33.493 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:33.571 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:33.571 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:33.572 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:33.572 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:33.572 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:33.572 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:33.572 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:33.572 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:33.573 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:33.573 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [100ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [161ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [162ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [163ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [163ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [241ms] refreshing cache from master, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [243ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@669924276(state=PENDING, result=null, callback=(continuation of Deferred@536328040 after retry RPC after error@1307073270) -> (continuation of Deferred@1808205096 after retry RPC after error@1025683583) -> (continuation of Deferred@533466180 after retry RPC after error@276613793) -> (continuation of Deferred@1128358851 after retry RPC after error@1808306844) -> (continuation of Deferred@345574797 after retry RPC after error@1648813181) -> (continuation of Deferred@718396557 after retry RPC after error@86177593) -> (continuation of Deferred@1493923701 after retry RPC after error@213587237), errback=(continuation of Deferred@536328040 after retry RPC after error@1307073270) -> (continuation of Deferred@1808205096 after retry RPC after error@1025683583) -> (continuation of Deferred@533466180 after retry RPC after error@276613793) -> (continuation of Deferred@1128358851 after retry RPC after error@1808306844) -> (continuation of Deferred@345574797 after retry RPC after error@1648813181) -> (continuation of Deferred@718396557 after retry RPC after error@86177593) -> (continuation of Deferred@1493923701 after retry RPC after error@213587237)))
20:28:33.573 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [100ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [161ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [162ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [163ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [163ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [241ms] refreshing cache from master, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [243ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@669924276(state=PENDING, result=null, callback=(continuation of Deferred@536328040 after retry RPC after error@1307073270) -> (continuation of Deferred@1808205096 after retry RPC after error@1025683583) -> (continuation of Deferred@533466180 after retry RPC after error@276613793) -> (continuation of Deferred@1128358851 after retry RPC after error@1808306844) -> (continuation of Deferred@345574797 after retry RPC after error@1648813181) -> (continuation of Deferred@718396557 after retry RPC after error@86177593) -> (continuation of Deferred@1493923701 after retry RPC after error@213587237), errback=(continuation of Deferred@536328040 after retry RPC after error@1307073270) -> (continuation of Deferred@1808205096 after retry RPC after error@1025683583) -> (continuation of Deferred@533466180 after retry RPC after error@276613793) -> (continuation of Deferred@1128358851 after retry RPC after error@1808306844) -> (continuation of Deferred@345574797 after retry RPC after error@1648813181) -> (continuation of Deferred@718396557 after retry RPC after error@86177593) -> (continuation of Deferred@1493923701 after retry RPC after error@213587237)))
20:28:33.591 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:33.591 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:33.591 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:33.591 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:33.591 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:33.592 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:33.592 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:33.592 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:33.592 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:33.611 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:33.611 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:33.611 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:33.611 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:33.611 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:33.611 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:33.611 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:33.611 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:33.612 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:33.630 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:33.631 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:33.631 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:33.631 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:33.631 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:33.631 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:33.631 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:33.631 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:33.632 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:33.650 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:33.650 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:33.651 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:33.651 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:33.651 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:33.651 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:33.651 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:33.651 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:33.652 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:33.671 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:33.671 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:33.671 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:33.671 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:33.671 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:33.672 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:33.672 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:33.672 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:33.672 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:33.691 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:33.691 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:33.691 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:33.691 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:33.691 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:33.691 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:33.691 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:33.691 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:33.692 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:33.750 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:33.751 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:33.751 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:33.751 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:33.751 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:33.751 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:33.752 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:33.752 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:33.752 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:33.752 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=161), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [0ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [1ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [20ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [40ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [60ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [80ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [99ms] refreshing cache from master, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [100ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [100ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [101ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [159ms] refreshing cache from master, [159ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [159ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [160ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [161ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@279814816(state=PENDING, result=null, callback=(continuation of Deferred@856281945 after retry RPC after error@21613362) -> (continuation of Deferred@1458191219 after retry RPC after error@732900268) -> (continuation of Deferred@1447147163 after retry RPC after error@421455674) -> (continuation of Deferred@505964233 after retry RPC after error@1275645163) -> (continuation of Deferred@388390559 after retry RPC after error@2037341302) -> (continuation of Deferred@615281596 after retry RPC after error@784960598), errback=(continuation of Deferred@856281945 after retry RPC after error@21613362) -> (continuation of Deferred@1458191219 after retry RPC after error@732900268) -> (continuation of Deferred@1447147163 after retry RPC after error@421455674) -> (continuation of Deferred@505964233 after retry RPC after error@1275645163) -> (continuation of Deferred@388390559 after retry RPC after error@2037341302) -> (continuation of Deferred@615281596 after retry RPC after error@784960598)))
20:28:33.753 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=161), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [0ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [1ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [20ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [40ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [60ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [80ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [99ms] refreshing cache from master, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [100ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [100ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [101ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [159ms] refreshing cache from master, [159ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [159ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [160ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [161ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@279814816(state=PENDING, result=null, callback=(continuation of Deferred@856281945 after retry RPC after error@21613362) -> (continuation of Deferred@1458191219 after retry RPC after error@732900268) -> (continuation of Deferred@1447147163 after retry RPC after error@421455674) -> (continuation of Deferred@505964233 after retry RPC after error@1275645163) -> (continuation of Deferred@388390559 after retry RPC after error@2037341302) -> (continuation of Deferred@615281596 after retry RPC after error@784960598), errback=(continuation of Deferred@856281945 after retry RPC after error@21613362) -> (continuation of Deferred@1458191219 after retry RPC after error@732900268) -> (continuation of Deferred@1447147163 after retry RPC after error@421455674) -> (continuation of Deferred@505964233 after retry RPC after error@1275645163) -> (continuation of Deferred@388390559 after retry RPC after error@2037341302) -> (continuation of Deferred@615281596 after retry RPC after error@784960598)))
20:28:33.770 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:33.770 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:33.771 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:33.771 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:33.771 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:33.771 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:33.771 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:33.771 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:33.772 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:33.791 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:33.791 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:33.791 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:33.791 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:33.791 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:33.791 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:33.791 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:33.791 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:33.792 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:33.810 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:33.811 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:33.811 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:33.811 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:33.811 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:33.811 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:33.811 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:33.811 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:33.812 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:33.831 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:33.831 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:33.831 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:33.831 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:33.831 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:33.831 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:33.831 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:33.831 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:33.832 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:33.850 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:33.850 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:33.851 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:33.851 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:33.851 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:33.851 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:33.851 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:33.851 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:33.852 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:33.890 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:33.890 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:33.891 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:33.891 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:33.891 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:33.891 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:33.891 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:33.891 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:33.892 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:33.970 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:33.972 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:33.972 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:33.972 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:33.972 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:33.972 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:33.972 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:33.973 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:33.973 [WARN - kudu-nio-6] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:34.011 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:34.011 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:34.011 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:34.012 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:34.012 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:34.012 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:34.012 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:34.012 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:34.012 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:34.013 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [202ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [203ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [203ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [241ms] refreshing cache from master, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, deferred=Deferred@1361160874(state=PENDING, result=null, callback=(continuation of Deferred@1433806721 after retry RPC after error@599572082) -> (continuation of Deferred@2070223556 after retry RPC after error@659393528) -> (continuation of Deferred@1130625727 after retry RPC after error@90866575) -> (continuation of Deferred@1443502812 after retry RPC after error@500386222) -> (continuation of Deferred@1102704297 after retry RPC after error@270429162) -> (continuation of Deferred@381968335 after retry RPC after error@291367552) -> (continuation of Deferred@522162852 after retry RPC after error@2084432790), errback=(continuation of Deferred@1433806721 after retry RPC after error@599572082) -> (continuation of Deferred@2070223556 after retry RPC after error@659393528) -> (continuation of Deferred@1130625727 after retry RPC after error@90866575) -> (continuation of Deferred@1443502812 after retry RPC after error@500386222) -> (continuation of Deferred@1102704297 after retry RPC after error@270429162) -> (continuation of Deferred@381968335 after retry RPC after error@291367552) -> (continuation of Deferred@522162852 after retry RPC after error@2084432790)))
20:28:34.013 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [122ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [202ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [203ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [203ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [241ms] refreshing cache from master, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, deferred=Deferred@1361160874(state=PENDING, result=null, callback=(continuation of Deferred@1433806721 after retry RPC after error@599572082) -> (continuation of Deferred@2070223556 after retry RPC after error@659393528) -> (continuation of Deferred@1130625727 after retry RPC after error@90866575) -> (continuation of Deferred@1443502812 after retry RPC after error@500386222) -> (continuation of Deferred@1102704297 after retry RPC after error@270429162) -> (continuation of Deferred@381968335 after retry RPC after error@291367552) -> (continuation of Deferred@522162852 after retry RPC after error@2084432790), errback=(continuation of Deferred@1433806721 after retry RPC after error@599572082) -> (continuation of Deferred@2070223556 after retry RPC after error@659393528) -> (continuation of Deferred@1130625727 after retry RPC after error@90866575) -> (continuation of Deferred@1443502812 after retry RPC after error@500386222) -> (continuation of Deferred@1102704297 after retry RPC after error@270429162) -> (continuation of Deferred@381968335 after retry RPC after error@291367552) -> (continuation of Deferred@522162852 after retry RPC after error@2084432790)))
20:28:34.031 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:34.031 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:34.031 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:34.032 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:34.032 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:34.032 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:34.032 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:34.032 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:34.032 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:34.051 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:34.051 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:34.051 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:34.051 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:34.051 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:34.051 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:34.051 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:34.051 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:34.052 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:34.070 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:34.071 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:34.071 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:34.071 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:34.071 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:34.071 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:34.071 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:34.071 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:34.072 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:34.090 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:34.090 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:34.091 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:34.091 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:34.091 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:34.091 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:34.091 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:34.091 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:34.091 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:34.111 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:34.111 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:34.112 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:34.112 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:34.112 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:34.112 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:34.112 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:34.112 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:34.112 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:34.131 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:34.131 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:34.131 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:34.131 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:34.131 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:34.131 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:34.132 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:34.132 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:34.132 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:34.210 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:34.210 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:34.211 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:34.211 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:34.211 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:34.211 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:34.211 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:34.211 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:34.212 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:34.250 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:34.250 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:34.251 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:34.251 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:34.251 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:34.251 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:34.251 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:34.251 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:34.252 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:34.271 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:34.271 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:34.272 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:34.272 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:34.272 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:34.272 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:34.272 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:34.272 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:34.272 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:34.273 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=241), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [1ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [20ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [40ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [60ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [101ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [179ms] refreshing cache from master, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [180ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [180ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [180ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [181ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [219ms] refreshing cache from master, [219ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [219ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [219ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [220ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [220ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [221ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [241ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, deferred=Deferred@292833328(state=PENDING, result=null, callback=(continuation of Deferred@121285040 after retry RPC after error@1002511209) -> (continuation of Deferred@88682407 after retry RPC after error@911201014) -> (continuation of Deferred@1693092518 after retry RPC after error@1472454401) -> (continuation of Deferred@465299017 after retry RPC after error@1395347370) -> (continuation of Deferred@1663324371 after retry RPC after error@718712929) -> (continuation of Deferred@1847296732 after retry RPC after error@1740953258) -> (continuation of Deferred@640135241 after retry RPC after error@963467604) -> (continuation of Deferred@189936071 after retry RPC after error@1247987585), errback=(continuation of Deferred@121285040 after retry RPC after error@1002511209) -> (continuation of Deferred@88682407 after retry RPC after error@911201014) -> (continuation of Deferred@1693092518 after retry RPC after error@1472454401) -> (continuation of Deferred@465299017 after retry RPC after error@1395347370) -> (continuation of Deferred@1663324371 after retry RPC after error@718712929) -> (continuation of Deferred@1847296732 after retry RPC after error@1740953258) -> (continuation of Deferred@640135241 after retry RPC after error@963467604) -> (continuation of Deferred@189936071 after retry RPC after error@1247987585)))
20:28:34.273 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=241), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [1ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [20ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [40ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [60ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [101ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [179ms] refreshing cache from master, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [180ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [180ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [180ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [181ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [219ms] refreshing cache from master, [219ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [219ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [219ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [220ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [220ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [221ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [241ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, deferred=Deferred@292833328(state=PENDING, result=null, callback=(continuation of Deferred@121285040 after retry RPC after error@1002511209) -> (continuation of Deferred@88682407 after retry RPC after error@911201014) -> (continuation of Deferred@1693092518 after retry RPC after error@1472454401) -> (continuation of Deferred@465299017 after retry RPC after error@1395347370) -> (continuation of Deferred@1663324371 after retry RPC after error@718712929) -> (continuation of Deferred@1847296732 after retry RPC after error@1740953258) -> (continuation of Deferred@640135241 after retry RPC after error@963467604) -> (continuation of Deferred@189936071 after retry RPC after error@1247987585), errback=(continuation of Deferred@121285040 after retry RPC after error@1002511209) -> (continuation of Deferred@88682407 after retry RPC after error@911201014) -> (continuation of Deferred@1693092518 after retry RPC after error@1472454401) -> (continuation of Deferred@465299017 after retry RPC after error@1395347370) -> (continuation of Deferred@1663324371 after retry RPC after error@718712929) -> (continuation of Deferred@1847296732 after retry RPC after error@1740953258) -> (continuation of Deferred@640135241 after retry RPC after error@963467604) -> (continuation of Deferred@189936071 after retry RPC after error@1247987585)))
20:28:34.291 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:34.291 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:34.291 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:34.291 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:34.291 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:34.291 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:34.291 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:34.292 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:34.292 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:34.310 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:34.311 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:34.311 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:34.311 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:34.311 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:34.311 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:34.311 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:34.311 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:34.312 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:34.330 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:34.330 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:34.331 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:34.331 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:34.331 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:34.331 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:34.331 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:34.331 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:34.331 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:34.331 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:34.332 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:34.332 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:34.351 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:34.351 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:34.352 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:34.352 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:34.352 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:34.352 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:34.352 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:34.352 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:34.353 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:34.371 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:34.371 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:34.371 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:34.372 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:34.372 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:34.372 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:34.372 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:34.372 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:34.372 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:34.391 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:34.391 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:34.391 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:34.391 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:34.391 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:34.392 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:34.392 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:34.392 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:34.392 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:34.451 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:34.451 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:34.452 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:34.452 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:34.452 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:34.452 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:34.452 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:34.452 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:34.453 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:34.453 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [63ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [161ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [163ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@1906228384(state=PENDING, result=null, callback=(continuation of Deferred@902640366 after retry RPC after error@1574490924) -> (continuation of Deferred@2106959157 after retry RPC after error@2037018011) -> (continuation of Deferred@627862042 after retry RPC after error@1043762216) -> (continuation of Deferred@1851402966 after retry RPC after error@1157010483) -> (continuation of Deferred@34343437 after retry RPC after error@139563432) -> (continuation of Deferred@937559790 after retry RPC after error@1204112173), errback=(continuation of Deferred@902640366 after retry RPC after error@1574490924) -> (continuation of Deferred@2106959157 after retry RPC after error@2037018011) -> (continuation of Deferred@627862042 after retry RPC after error@1043762216) -> (continuation of Deferred@1851402966 after retry RPC after error@1157010483) -> (continuation of Deferred@34343437 after retry RPC after error@139563432) -> (continuation of Deferred@937559790 after retry RPC after error@1204112173)))
20:28:34.454 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [63ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [161ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [163ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@1906228384(state=PENDING, result=null, callback=(continuation of Deferred@902640366 after retry RPC after error@1574490924) -> (continuation of Deferred@2106959157 after retry RPC after error@2037018011) -> (continuation of Deferred@627862042 after retry RPC after error@1043762216) -> (continuation of Deferred@1851402966 after retry RPC after error@1157010483) -> (continuation of Deferred@34343437 after retry RPC after error@139563432) -> (continuation of Deferred@937559790 after retry RPC after error@1204112173), errback=(continuation of Deferred@902640366 after retry RPC after error@1574490924) -> (continuation of Deferred@2106959157 after retry RPC after error@2037018011) -> (continuation of Deferred@627862042 after retry RPC after error@1043762216) -> (continuation of Deferred@1851402966 after retry RPC after error@1157010483) -> (continuation of Deferred@34343437 after retry RPC after error@139563432) -> (continuation of Deferred@937559790 after retry RPC after error@1204112173)))
20:28:34.471 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:34.471 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:34.472 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:34.472 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:34.472 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:34.472 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:34.472 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:34.472 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:34.473 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:34.491 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:34.491 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:34.491 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:34.491 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:34.492 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:34.492 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:34.492 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:34.492 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:34.493 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:34.511 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:34.511 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:34.511 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:34.511 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:34.511 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:34.511 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:34.511 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:34.511 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:34.513 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:34.530 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:34.531 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:34.531 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:34.531 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:34.531 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:34.531 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:34.531 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:34.531 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:34.532 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:34.550 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:34.550 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:34.551 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:34.551 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:34.551 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:34.551 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:34.551 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:34.551 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:34.552 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:34.571 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:34.571 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:34.571 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:34.572 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:34.572 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:34.572 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:34.572 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:34.572 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:34.572 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:34.631 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:34.631 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:34.632 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:34.632 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:34.632 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:34.632 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:34.632 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:34.632 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:34.633 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:34.671 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:34.671 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:34.671 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:34.671 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:34.672 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:34.672 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:34.672 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:34.672 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:34.672 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:34.711 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:34.711 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:34.711 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:34.711 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:34.712 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:34.712 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:34.712 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:34.712 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:34.712 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:34.731 [DEBUG - pool-40-thread-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=259), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [40ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [60ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [60ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [80ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [101ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [201ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [241ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [241ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, deferred=Deferred@743470336(state=PENDING, result=null, callback=(continuation of Deferred@507156988 after retry RPC after error@1021799378) -> (continuation of Deferred@723265238 after retry RPC after error@80457408) -> (continuation of Deferred@1316233948 after retry RPC after error@1238001979) -> (continuation of Deferred@53140546 after retry RPC after error@1523507017) -> (continuation of Deferred@893608974 after retry RPC after error@802836524) -> (continuation of Deferred@2059882841 after retry RPC after error@1449127391) -> (continuation of Deferred@2014482446 after retry RPC after error@95745040) -> (continuation of Deferred@1161375846 after retry RPC after error@1822497803) -> (continuation of Deferred@14644081 after retry RPC after error@585700110), errback=(continuation of Deferred@507156988 after retry RPC after error@1021799378) -> (continuation of Deferred@723265238 after retry RPC after error@80457408) -> (continuation of Deferred@1316233948 after retry RPC after error@1238001979) -> (continuation of Deferred@53140546 after retry RPC after error@1523507017) -> (continuation of Deferred@893608974 after retry RPC after error@802836524) -> (continuation of Deferred@2059882841 after retry RPC after error@1449127391) -> (continuation of Deferred@2014482446 after retry RPC after error@95745040) -> (continuation of Deferred@1161375846 after retry RPC after error@1822497803) -> (continuation of Deferred@14644081 after retry RPC after error@585700110)))
20:28:34.731 [WARN - pool-40-thread-1] (HashedWheelTimer.java:716) An exception was thrown by TimerTask.
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713)
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701)
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:787)
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501)
at java.base/java.lang.Thread.run(Thread.java:840)
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713)
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701)
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:787)
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501)
at java.base/java.lang.Thread.run(Thread.java:840)
20:28:34.751 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:34.751 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:34.751 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:34.752 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:34.752 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:34.752 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:34.752 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:34.752 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:34.753 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:34.771 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:34.771 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:34.771 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:34.771 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:34.771 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:34.772 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:34.772 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:34.772 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:34.772 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:34.790 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:34.791 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:34.791 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:34.791 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:34.791 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:34.791 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:34.791 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:34.791 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:34.792 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:34.810 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:34.810 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:34.811 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:34.811 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:34.811 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:34.811 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:34.811 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:34.811 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:34.812 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:34.831 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:34.831 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:34.831 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:34.831 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:34.831 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:34.831 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:34.832 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:34.832 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:34.832 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:34.871 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:34.871 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:34.871 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:34.871 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:34.871 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:34.871 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:34.871 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:34.871 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:34.872 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:34.931 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:34.931 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:34.931 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:34.931 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:34.932 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:34.932 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:34.932 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:34.932 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:34.932 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:34.933 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=181), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [40ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [60ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [80ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [119ms] refreshing cache from master, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [120ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [120ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [121ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [180ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@416398739(state=PENDING, result=null, callback=(continuation of Deferred@561762755 after retry RPC after error@678254522) -> (continuation of Deferred@163497399 after retry RPC after error@471256118) -> (continuation of Deferred@1671230398 after retry RPC after error@554608880) -> (continuation of Deferred@1591886363 after retry RPC after error@1950302861) -> (continuation of Deferred@1891262135 after retry RPC after error@614936952) -> (continuation of Deferred@1746492781 after retry RPC after error@1715229129), errback=(continuation of Deferred@561762755 after retry RPC after error@678254522) -> (continuation of Deferred@163497399 after retry RPC after error@471256118) -> (continuation of Deferred@1671230398 after retry RPC after error@554608880) -> (continuation of Deferred@1591886363 after retry RPC after error@1950302861) -> (continuation of Deferred@1891262135 after retry RPC after error@614936952) -> (continuation of Deferred@1746492781 after retry RPC after error@1715229129)))
20:28:34.933 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=181), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [40ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [40ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [60ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [80ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [119ms] refreshing cache from master, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [120ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [120ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [121ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [180ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@416398739(state=PENDING, result=null, callback=(continuation of Deferred@561762755 after retry RPC after error@678254522) -> (continuation of Deferred@163497399 after retry RPC after error@471256118) -> (continuation of Deferred@1671230398 after retry RPC after error@554608880) -> (continuation of Deferred@1591886363 after retry RPC after error@1950302861) -> (continuation of Deferred@1891262135 after retry RPC after error@614936952) -> (continuation of Deferred@1746492781 after retry RPC after error@1715229129), errback=(continuation of Deferred@561762755 after retry RPC after error@678254522) -> (continuation of Deferred@163497399 after retry RPC after error@471256118) -> (continuation of Deferred@1671230398 after retry RPC after error@554608880) -> (continuation of Deferred@1591886363 after retry RPC after error@1950302861) -> (continuation of Deferred@1891262135 after retry RPC after error@614936952) -> (continuation of Deferred@1746492781 after retry RPC after error@1715229129)))
20:28:34.951 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:34.951 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:34.951 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:34.951 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:34.952 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:34.952 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:34.952 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:34.952 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:34.952 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:34.971 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:34.971 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:34.971 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:34.971 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:34.971 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:34.971 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:34.971 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:34.972 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:34.972 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:34.990 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:34.991 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:34.991 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:34.991 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:34.991 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:34.991 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:34.991 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:34.991 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:34.992 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:35.010 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:35.010 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:35.011 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:35.011 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:35.011 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:35.011 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:35.011 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:35.011 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:35.012 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:35.031 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:35.031 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:35.031 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:35.031 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:35.032 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:35.032 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:35.032 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:35.032 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:35.032 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:35.071 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:35.071 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:35.071 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:35.071 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:35.071 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:35.071 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:35.071 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:35.071 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:35.072 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:35.090 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:35.091 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:35.091 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:35.091 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:35.091 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:35.091 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:35.091 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:35.091 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:35.091 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:35.091 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:35.092 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:35.092 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:35.171 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:35.171 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:35.171 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:35.171 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:35.171 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:35.171 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:35.171 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:35.172 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:35.172 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:35.172 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=221), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [1ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [20ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [40ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [60ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [119ms] refreshing cache from master, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [120ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [120ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [121ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [140ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [140ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [141ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [219ms] refreshing cache from master, [219ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [219ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [220ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [221ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@884877613(state=PENDING, result=null, callback=(continuation of Deferred@1812726951 after retry RPC after error@271154595) -> (continuation of Deferred@1563335697 after retry RPC after error@412885464) -> (continuation of Deferred@347682640 after retry RPC after error@1336316929) -> (continuation of Deferred@1715558384 after retry RPC after error@522860545) -> (continuation of Deferred@1123323651 after retry RPC after error@956894656) -> (continuation of Deferred@1427841990 after retry RPC after error@465560404) -> (continuation of Deferred@2102782903 after retry RPC after error@205758160), errback=(continuation of Deferred@1812726951 after retry RPC after error@271154595) -> (continuation of Deferred@1563335697 after retry RPC after error@412885464) -> (continuation of Deferred@347682640 after retry RPC after error@1336316929) -> (continuation of Deferred@1715558384 after retry RPC after error@522860545) -> (continuation of Deferred@1123323651 after retry RPC after error@956894656) -> (continuation of Deferred@1427841990 after retry RPC after error@465560404) -> (continuation of Deferred@2102782903 after retry RPC after error@205758160)))
20:28:35.173 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=221), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [1ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [20ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [20ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [21ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [40ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [40ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [60ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [60ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [119ms] refreshing cache from master, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [120ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [120ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [121ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [121ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [140ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [140ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [141ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [141ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [219ms] refreshing cache from master, [219ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [219ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [220ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [221ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, deferred=Deferred@884877613(state=PENDING, result=null, callback=(continuation of Deferred@1812726951 after retry RPC after error@271154595) -> (continuation of Deferred@1563335697 after retry RPC after error@412885464) -> (continuation of Deferred@347682640 after retry RPC after error@1336316929) -> (continuation of Deferred@1715558384 after retry RPC after error@522860545) -> (continuation of Deferred@1123323651 after retry RPC after error@956894656) -> (continuation of Deferred@1427841990 after retry RPC after error@465560404) -> (continuation of Deferred@2102782903 after retry RPC after error@205758160), errback=(continuation of Deferred@1812726951 after retry RPC after error@271154595) -> (continuation of Deferred@1563335697 after retry RPC after error@412885464) -> (continuation of Deferred@347682640 after retry RPC after error@1336316929) -> (continuation of Deferred@1715558384 after retry RPC after error@522860545) -> (continuation of Deferred@1123323651 after retry RPC after error@956894656) -> (continuation of Deferred@1427841990 after retry RPC after error@465560404) -> (continuation of Deferred@2102782903 after retry RPC after error@205758160)))
20:28:35.190 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:35.191 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:35.191 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:35.191 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:35.191 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:35.191 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:35.191 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:35.191 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:35.192 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:35.210 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:35.210 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:35.211 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:35.211 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:35.211 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:35.211 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:35.211 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:35.211 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:35.212 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:35.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:28:35.219022 4132 master_runner.cc:186] Error getting master registration for 127.3.46.61:39407: Network error: Client connection negotiation failed: client connection to 127.3.46.61:39407: connect: Connection refused (error 111),
20:28:35.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:28:35.219194 4132 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
20:28:35.231 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:35.231 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:35.231 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:35.231 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:35.232 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:35.232 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:35.232 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:35.232 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:35.232 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:35.251 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:35.251 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:35.251 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:35.251 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:35.251 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:35.251 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:35.251 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:35.251 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:35.252 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:35.270 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:35.271 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:35.271 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:35.271 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:35.271 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:35.271 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:35.271 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:35.271 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:35.272 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:35.290 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.61:39407(127.3.46.61:39407)] connecting to peer
20:28:35.290 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.46.60:36127(127.3.46.60:36127)] connecting to peer
20:28:35.291 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:35.291 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.3.46.61:39407(127.3.46.61:39407)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:35.291 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:35.291 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:35.291 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.61:39407: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407
20:28:35.291 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.3.46.60:36127(127.3.46.60:36127)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:35.291 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:35.291 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.3.46.60:36127: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127
20:28:35.292 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:35.292 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=GetTransactionState, tablet=null, attempt=35, TimeoutTracker(timeout=50000, elapsed=48117), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [3ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [17ms] refreshing cache from master, [18ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [18ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [18ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [18ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [18ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [19ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [19ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [37ms] refreshing cache from master, [37ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [37ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [37ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [37ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [37ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [39ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [39ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [57ms] refreshing cache from master, [58ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [58ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [58ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [58ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [58ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [59ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [59ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [77ms] refreshing cache from master, [77ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [77ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [78ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [78ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [78ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [79ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [79ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [116ms] refreshing cache from master, [117ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [117ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [117ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [118ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [119ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [119ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [120ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [176ms] refreshing cache from master, [177ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [177ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [177ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [178ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [178ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [178ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [179ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [216ms] refreshing cache from master, [216ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [216ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [217ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [217ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [218ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [218ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [218ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [436ms] refreshing cache from master, [437ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [437ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [437ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [438ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [438ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [439ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [439ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [876ms] refreshing cache from master, [877ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [877ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [878ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [879ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [879ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [879ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [880ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1296ms] refreshing cache from master, [1296ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [1296ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1297ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1298ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1298ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [1298ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [1298ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [1636ms] refreshing cache from master, [1636ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [1637ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1637ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1638ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1638ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [1638ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [1638ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [3936ms] refreshing cache from master, [3936ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [3937ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [3937ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [3938ms] too many traces: truncated at 100 traces, deferred=Deferred@1923540560(state=PENDING, result=null, callback=(continuation of Deferred@1307506784 after retry RPC after error@1917288227) -> (continuation of Deferred@1889415068 after retry RPC after error@902808505) -> (continuation of Deferred@265818264 after retry RPC after error@1177536551) -> (continuation of Deferred@1204612529 after retry RPC after error@206912762) -> (continuation of Deferred@1137094968 after retry RPC after error@2073048983) -> (continuation of Deferred@1527901233 after retry RPC after error@2114365765) -> (continuation of Deferred@146774220 after retry RPC after error@1927883459) -> (continuation of Deferred@650977483 after retry RPC after error@1842018774) -> (continuation of Deferred@432498011 after retry RPC after error@583269266) -> (continuation of Deferred@1841777785 after retry RPC after error@1467554910) -> (continuation of Deferred@1966491223 after retry RPC after error@1209281544) -> (continuation of Deferred@526849527 after retry RPC after error@6513065) -> (continuation of Deferred@1193315084 after retry RPC after error@775785356) -> (continuation of Deferred@164821777 after retry RPC after error@304590251) -> (continuation of Deferred@1369193480 after retry RPC after error@331328604) -> (continuation of Deferred@229361481 after retry RPC after error@583952802) -> (continuation of Deferred@310541711 after retry RPC after error@2083651279) -> (continuation of Deferred@796131215 after retry RPC after error@2116092090) -> (continuation of Deferred@2144236788 after retry RPC after error@209336900) -> (continuation of Deferred@1011168195 after retry RPC after error@1251463201) -> (continuation of Deferred@1190290515 after retry RPC after error@461889235) -> (continuation of Deferred@248069451 after retry RPC after error@2049700459) -> (continuation of Deferred@791593345 after retry RPC after error@184218609) -> (continuation of Deferred@1302242432 after retry RPC after error@1753460722) -> (continuation of Deferred@838355370 after retry RPC after error@1824543007) -> (continuation of Deferred@1413628579 after retry RPC after error@566987750) -> (continuation of Deferred@1384981356 after retry RPC after error@1017895051) -> (continuation of Deferred@2059059943 after retry RPC after error@122942189) -> (continuation of Deferred@1014904616 after retry RPC after error@1593040812) -> (continuation of Deferred@1920111471 after retry RPC after error@1789842262) -> (continuation of Deferred@665091676 after retry RPC after error@1743440827) -> (continuation of Deferred@23446978 after retry RPC after error@500011739) -> (continuation of Deferred@429869472 after retry RPC after error@672792123) -> (continuation of Deferred@125662511 after retry RPC after error@1993831505), errback=(continuation of Deferred@1307506784 after retry RPC after error@1917288227) -> (continuation of Deferred@1889415068 after retry RPC after error@902808505) -> (continuation of Deferred@265818264 after retry RPC after error@1177536551) -> (continuation of Deferred@1204612529 after retry RPC after error@206912762) -> (continuation of Deferred@1137094968 after retry RPC after error@2073048983) -> (continuation of Deferred@1527901233 after retry RPC after error@2114365765) -> (continuation of Deferred@146774220 after retry RPC after error@1927883459) -> (continuation of Deferred@650977483 after retry RPC after error@1842018774) -> (continuation of Deferred@432498011 after retry RPC after error@583269266) -> (continuation of Deferred@1841777785 after retry RPC after error@1467554910) -> (continuation of Deferred@1966491223 after retry RPC after error@1209281544) -> (continuation of Deferred@526849527 after retry RPC after error@6513065) -> (continuation of Deferred@1193315084 after retry RPC after error@775785356) -> (continuation of Deferred@164821777 after retry RPC after error@304590251) -> (continuation of Deferred@1369193480 after retry RPC after error@331328604) -> (continuation of Deferred@229361481 after retry RPC after error@583952802) -> (continuation of Deferred@310541711 after retry RPC after error@2083651279) -> (continuation of Deferred@796131215 after retry RPC after error@2116092090) -> (continuation of Deferred@2144236788 after retry RPC after error@209336900) -> (continuation of Deferred@1011168195 after retry RPC after error@1251463201) -> (continuation of Deferred@1190290515 after retry RPC after error@461889235) -> (continuation of Deferred@248069451 after retry RPC after error@2049700459) -> (continuation of Deferred@791593345 after retry RPC after error@184218609) -> (continuation of Deferred@1302242432 after retry RPC after error@1753460722) -> (continuation of Deferred@838355370 after retry RPC after error@1824543007) -> (continuation of Deferred@1413628579 after retry RPC after error@566987750) -> (continuation of Deferred@1384981356 after retry RPC after error@1017895051) -> (continuation of Deferred@2059059943 after retry RPC after error@122942189) -> (continuation of Deferred@1014904616 after retry RPC after error@1593040812) -> (continuation of Deferred@1920111471 after retry RPC after error@1789842262) -> (continuation of Deferred@665091676 after retry RPC after error@1743440827) -> (continuation of Deferred@23446978 after retry RPC after error@500011739) -> (continuation of Deferred@429869472 after retry RPC after error@672792123) -> (continuation of Deferred@125662511 after retry RPC after error@1993831505)))
20:28:35.293 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127; will retry
20:28:35.294 [DEBUG - main] (Connection.java:575) [peer master-127.3.46.62:46071(127.3.46.62:46071)] disconnecting while in state READY
20:28:35.294 [DEBUG - main] (Connection.java:575) [peer master-127.3.46.60:36127(127.3.46.60:36127)] disconnecting while in state TERMINATED
20:28:35.294 [DEBUG - main] (Connection.java:575) [peer d00c04330aef4a87a22435384a1a5c12(127.3.46.1:37067)] disconnecting while in state READY
20:28:35.294 [DEBUG - main] (Connection.java:575) [peer master-127.3.46.61:39407(127.3.46.61:39407)] disconnecting while in state TERMINATED
20:28:35.295 [DEBUG - kudu-nio-6] (AsyncKuduClient.java:2704) Releasing all remaining resources
20:28:35.295 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.3.46.62:46071(127.3.46.62:46071)] handling channelInactive
20:28:35.295 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.46.62:46071(127.3.46.62:46071)] cleaning up while in state READY due to: connection closed
20:28:35.295 [DEBUG - kudu-nio-6] (Connection.java:239) [peer d00c04330aef4a87a22435384a1a5c12(127.3.46.1:37067)] handling channelInactive
20:28:35.295 [DEBUG - kudu-nio-6] (Connection.java:709) [peer d00c04330aef4a87a22435384a1a5c12(127.3.46.1:37067)] cleaning up while in state READY due to: connection closed
20:28:35.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:28:35.296092 3256 external_mini_cluster.cc:1699] Attempting to check leaks for /tmp/dist-test-taska94UBC/build/asan/bin/kudu pid 3549
20:28:35.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:28:35.308498 3701 generic_service.cc:196] Checking for leaks (request via RPC)
20:28:35.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:28:35.434499 3256 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taska94UBC/build/asan/bin/kudu with pid 3549
20:28:35.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:28:35.452634 3728 connection.cc:570] server connection from 127.3.46.1:40617 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
20:28:35.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:28:35.452986 3256 external_mini_cluster.cc:1699] Attempting to check leaks for /tmp/dist-test-taska94UBC/build/asan/bin/kudu pid 3711
20:28:35.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:28:35.460209 3843 generic_service.cc:196] Checking for leaks (request via RPC)
20:28:35.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:28:35.567528 3256 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taska94UBC/build/asan/bin/kudu with pid 3711
20:28:35.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:28:35.584151 3256 external_mini_cluster.cc:1699] Attempting to check leaks for /tmp/dist-test-taska94UBC/build/asan/bin/kudu pid 3853
20:28:35.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:28:35.591684 3985 generic_service.cc:196] Checking for leaks (request via RPC)
20:28:35.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:28:35.749459 3870 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.3.46.2:37949: connect: Connection refused (error 111)
20:28:35.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:28:35.749615 3256 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taska94UBC/build/asan/bin/kudu with pid 3853
20:28:35.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260501 20:28:35.773098 4148 connection.cc:570] server connection from 127.3.46.3:42519 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
20:28:35.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:28:35.773425 3256 external_mini_cluster.cc:1699] Attempting to check leaks for /tmp/dist-test-taska94UBC/build/asan/bin/kudu pid 4132
20:28:35.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:28:35.780278 4204 generic_service.cc:196] Checking for leaks (request via RPC)
20:28:35.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260501 20:28:35.929611 3256 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taska94UBC/build/asan/bin/kudu with pid 4132
20:28:35.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-05-01T20:28:35Z chronyd exiting
20:28:36.099 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestKuduTransaction.testTxnKeepaliveRollingSwitchToOtherTxnManager: failed attempt 1
java.lang.AssertionError: Timed out: cannot complete before timeout: KuduRpc(method=GetTransactionState, tablet=null, attempt=35, TimeoutTracker(timeout=50000, elapsed=48117), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [3ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [17ms] refreshing cache from master, [18ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [18ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [18ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [18ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [18ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [19ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [19ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [37ms] refreshing cache from master, [37ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [37ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [37ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [37ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [37ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [39ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [39ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [57ms] refreshing cache from master, [58ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [58ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [58ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [58ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [58ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [59ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [59ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [77ms] refreshing cache from master, [77ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [77ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [78ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [78ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [78ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [79ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [79ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [116ms] refreshing cache from master, [117ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [117ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [117ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [118ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [119ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [119ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [120ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [176ms] refreshing cache from master, [177ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [177ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [177ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [178ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [178ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [178ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [179ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [216ms] refreshing cache from master, [216ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [216ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [217ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [217ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [218ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [218ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [218ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [436ms] refreshing cache from master, [437ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [437ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [437ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [438ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [438ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [439ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [439ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [876ms] refreshing cache from master, [877ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [877ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [878ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [879ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [879ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [879ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [880ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1296ms] refreshing cache from master, [1296ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [1296ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1297ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1298ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1298ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [1298ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [1298ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [1636ms] refreshing cache from master, [1636ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [1637ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1637ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1638ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1638ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [1638ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [1638ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [3936ms] refreshing cache from master, [3936ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [3937ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [3937ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [3938ms] too many traces: truncated at 100 traces, deferred=Deferred@1923540560(state=PENDING, result=null, callback=(continuation of Deferred@1307506784 after retry RPC after error@1917288227) -> (continuation of Deferred@1889415068 after retry RPC after error@902808505) -> (continuation of Deferred@265818264 after retry RPC after error@1177536551) -> (continuation of Deferred@1204612529 after retry RPC after error@206912762) -> (continuation of Deferred@1137094968 after retry RPC after error@2073048983) -> (continuation of Deferred@1527901233 after retry RPC after error@2114365765) -> (continuation of Deferred@146774220 after retry RPC after error@1927883459) -> (continuation of Deferred@650977483 after retry RPC after error@1842018774) -> (continuation of Deferred@432498011 after retry RPC after error@583269266) -> (continuation of Deferred@1841777785 after retry RPC after error@1467554910) -> (continuation of Deferred@1966491223 after retry RPC after error@1209281544) -> (continuation of Deferred@526849527 after retry RPC after error@6513065) -> (continuation of Deferred@1193315084 after retry RPC after error@775785356) -> (continuation of Deferred@164821777 after retry RPC after error@304590251) -> (continuation of Deferred@1369193480 after retry RPC after error@331328604) -> (continuation of Deferred@229361481 after retry RPC after error@583952802) -> (continuation of Deferred@310541711 after retry RPC after error@2083651279) -> (continuation of Deferred@796131215 after retry RPC after error@2116092090) -> (continuation of Deferred@2144236788 after retry RPC after error@209336900) -> (continuation of Deferred@1011168195 after retry RPC after error@1251463201) -> (continuation of Deferred@1190290515 after retry RPC after error@461889235) -> (continuation of Deferred@248069451 after retry RPC after error@2049700459) -> (continuation of Deferred@791593345 after retry RPC after error@184218609) -> (continuation of Deferred@1302242432 after retry RPC after error@1753460722) -> (continuation of Deferred@838355370 after retry RPC after error@1824543007) -> (continuation of Deferred@1413628579 after retry RPC after error@566987750) -> (continuation of Deferred@1384981356 after retry RPC after error@1017895051) -> (continuation of Deferred@2059059943 after retry RPC after error@122942189) -> (continuation of Deferred@1014904616 after retry RPC after error@1593040812) -> (continuation of Deferred@1920111471 after retry RPC after error@1789842262) -> (continuation of Deferred@665091676 after retry RPC after error@1743440827) -> (continuation of Deferred@23446978 after retry RPC after error@500011739) -> (continuation of Deferred@429869472 after retry RPC after error@672792123) -> (continuation of Deferred@125662511 after retry RPC after error@1993831505), errback=(continuation of Deferred@1307506784 after retry RPC after error@1917288227) -> (continuation of Deferred@1889415068 after retry RPC after error@902808505) -> (continuation of Deferred@265818264 after retry RPC after error@1177536551) -> (continuation of Deferred@1204612529 after retry RPC after error@206912762) -> (continuation of Deferred@1137094968 after retry RPC after error@2073048983) -> (continuation of Deferred@1527901233 after retry RPC after error@2114365765) -> (continuation of Deferred@146774220 after retry RPC after error@1927883459) -> (continuation of Deferred@650977483 after retry RPC after error@1842018774) -> (continuation of Deferred@432498011 after retry RPC after error@583269266) -> (continuation of Deferred@1841777785 after retry RPC after error@1467554910) -> (continuation of Deferred@1966491223 after retry RPC after error@1209281544) -> (continuation of Deferred@526849527 after retry RPC after error@6513065) -> (continuation of Deferred@1193315084 after retry RPC after error@775785356) -> (continuation of Deferred@164821777 after retry RPC after error@304590251) -> (continuation of Deferred@1369193480 after retry RPC after error@331328604) -> (continuation of Deferred@229361481 after retry RPC after error@583952802) -> (continuation of Deferred@310541711 after retry RPC after error@2083651279) -> (continuation of Deferred@796131215 after retry RPC after error@2116092090) -> (continuation of Deferred@2144236788 after retry RPC after error@209336900) -> (continuation of Deferred@1011168195 after retry RPC after error@1251463201) -> (continuation of Deferred@1190290515 after retry RPC after error@461889235) -> (continuation of Deferred@248069451 after retry RPC after error@2049700459) -> (continuation of Deferred@791593345 after retry RPC after error@184218609) -> (continuation of Deferred@1302242432 after retry RPC after error@1753460722) -> (continuation of Deferred@838355370 after retry RPC after error@1824543007) -> (continuation of Deferred@1413628579 after retry RPC after error@566987750) -> (continuation of Deferred@1384981356 after retry RPC after error@1017895051) -> (continuation of Deferred@2059059943 after retry RPC after error@122942189) -> (continuation of Deferred@1014904616 after retry RPC after error@1593040812) -> (continuation of Deferred@1920111471 after retry RPC after error@1789842262) -> (continuation of Deferred@665091676 after retry RPC after error@1743440827) -> (continuation of Deferred@23446978 after retry RPC after error@500011739) -> (continuation of Deferred@429869472 after retry RPC after error@672792123) -> (continuation of Deferred@125662511 after retry RPC after error@1993831505)))
at org.junit.Assert.fail(Assert.java:89)
at org.junit.Assert.assertTrue(Assert.java:42)
at org.apache.kudu.client.TestKuduTransaction.testTxnKeepaliveRollingSwitchToOtherTxnManager(TestKuduTransaction.java:1214)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:569)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
at java.base/java.lang.Thread.run(Thread.java:840)
java.lang.AssertionError: Timed out: cannot complete before timeout: KuduRpc(method=GetTransactionState, tablet=null, attempt=35, TimeoutTracker(timeout=50000, elapsed=48117), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [3ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [17ms] refreshing cache from master, [18ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [18ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [18ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [18ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [18ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [19ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [19ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [37ms] refreshing cache from master, [37ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [37ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [37ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [37ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [37ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [39ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [39ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [57ms] refreshing cache from master, [58ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [58ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [58ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [58ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [58ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [59ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [59ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [77ms] refreshing cache from master, [77ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [77ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [78ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [78ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [78ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [79ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [79ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [116ms] refreshing cache from master, [117ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [117ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [117ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [118ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [119ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [119ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [120ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [176ms] refreshing cache from master, [177ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [177ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [177ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [178ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [178ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [178ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [179ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [216ms] refreshing cache from master, [216ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [216ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [217ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [217ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [218ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [218ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [218ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [436ms] refreshing cache from master, [437ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [437ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [437ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [438ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [438ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [439ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [439ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [876ms] refreshing cache from master, [877ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [877ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [878ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [879ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [879ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [879ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [880ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1296ms] refreshing cache from master, [1296ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [1296ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1297ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1298ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1298ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [1298ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [1298ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [1636ms] refreshing cache from master, [1636ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [1637ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [1637ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [1638ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.61:39407: Network error: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407, [1638ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.60:36127: Network error: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [1638ms] Sub RPC ConnectToMaster: received response from server master-127.3.46.62:46071: OK, [1638ms] delaying RPC due to: Service unavailable: Master config (127.3.46.62:46071,127.3.46.61:39407,127.3.46.60:36127) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.61:39407(127.3.46.61:39407): Connection refused: /127.3.46.61:39407,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.46.60:36127(127.3.46.60:36127): Connection refused: /127.3.46.60:36127, [3936ms] refreshing cache from master, [3936ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.62:46071, [3937ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.61:39407, [3937ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.46.60:36127, [3938ms] too many traces: truncated at 100 traces, deferred=Deferred@1923540560(state=PENDING, result=null, callback=(continuation of Deferred@1307506784 after retry RPC after error@1917288227) -> (continuation of Deferred@1889415068 after retry RPC after error@902808505) -> (continuation of Deferred@265818264 after retry RPC after error@1177536551) -> (continuation of Deferred@1204612529 after retry RPC after error@206912762) -> (continuation of Deferred@1137094968 after retry RPC after error@2073048983) -> (continuation of Deferred@1527901233 after retry RPC after error@2114365765) -> (continuation of Deferred@146774220 after retry RPC after error@1927883459) -> (continuation of Deferred@650977483 after retry RPC after error@1842018774) -> (continuation of Deferred@432498011 after retry RPC after error@583269266) -> (continuation of Deferred@1841777785 after retry RPC after error@1467554910) -> (continuation of Deferred@1966491223 after retry RPC after error@1209281544) -> (continuation of Deferred@526849527 after retry RPC after error@6513065) -> (continuation of Deferred@1193315084 after retry RPC after error@775785356) -> (continuation of Deferred@164821777 after retry RPC after error@304590251) -> (continuation of Deferred@1369193480 after retry RPC after error@331328604) -> (continuation of Deferred@229361481 after retry RPC after error@583952802) -> (continuation of Deferred@310541711 after retry RPC after error@2083651279) -> (continuation of Deferred@796131215 after retry RPC after error@2116092090) -> (continuation of Deferred@2144236788 after retry RPC after error@209336900) -> (continuation of Deferred@1011168195 after retry RPC after error@1251463201) -> (continuation of Deferred@1190290515 after retry RPC after error@461889235) -> (continuation of Deferred@248069451 after retry RPC after error@2049700459) -> (continuation of Deferred@791593345 after retry RPC after error@184218609) -> (continuation of Deferred@1302242432 after retry RPC after error@1753460722) -> (continuation of Deferred@838355370 after retry RPC after error@1824543007) -> (continuation of Deferred@1413628579 after retry RPC after error@566987750) -> (continuation of Deferred@1384981356 after retry RPC after error@1017895051) -> (continuation of Deferred@2059059943 after retry RPC after error@122942189) -> (continuation of Deferred@1014904616 after retry RPC after error@1593040812) -> (continuation of Deferred@1920111471 after retry RPC after error@1789842262) -> (continuation of Deferred@665091676 after retry RPC after error@1743440827) -> (continuation of Deferred@23446978 after retry RPC after error@500011739) -> (continuation of Deferred@429869472 after retry RPC after error@672792123) -> (continuation of Deferred@125662511 after retry RPC after error@1993831505), errback=(continuation of Deferred@1307506784 after retry RPC after error@1917288227) -> (continuation of Deferred@1889415068 after retry RPC after error@902808505) -> (continuation of Deferred@265818264 after retry RPC after error@1177536551) -> (continuation of Deferred@1204612529 after retry RPC after error@206912762) -> (continuation of Deferred@1137094968 after retry RPC after error@2073048983) -> (continuation of Deferred@1527901233 after retry RPC after error@2114365765) -> (continuation of Deferred@146774220 after retry RPC after error@1927883459) -> (continuation of Deferred@650977483 after retry RPC after error@1842018774) -> (continuation of Deferred@432498011 after retry RPC after error@583269266) -> (continuation of Deferred@1841777785 after retry RPC after error@1467554910) -> (continuation of Deferred@1966491223 after retry RPC after error@1209281544) -> (continuation of Deferred@526849527 after retry RPC after error@6513065) -> (continuation of Deferred@1193315084 after retry RPC after error@775785356) -> (continuation of Deferred@164821777 after retry RPC after error@304590251) -> (continuation of Deferred@1369193480 after retry RPC after error@331328604) -> (continuation of Deferred@229361481 after retry RPC after error@583952802) -> (continuation of Deferred@310541711 after retry RPC after error@2083651279) -> (continuation of Deferred@796131215 after retry RPC after error@2116092090) -> (continuation of Deferred@2144236788 after retry RPC after error@209336900) -> (continuation of Deferred@1011168195 after retry RPC after error@1251463201) -> (continuation of Deferred@1190290515 after retry RPC after error@461889235) -> (continuation of Deferred@248069451 after retry RPC after error@2049700459) -> (continuation of Deferred@791593345 after retry RPC after error@184218609) -> (continuation of Deferred@1302242432 after retry RPC after error@1753460722) -> (continuation of Deferred@838355370 after retry RPC after error@1824543007) -> (continuation of Deferred@1413628579 after retry RPC after error@566987750) -> (continuation of Deferred@1384981356 after retry RPC after error@1017895051) -> (continuation of Deferred@2059059943 after retry RPC after error@122942189) -> (continuation of Deferred@1014904616 after retry RPC after error@1593040812) -> (continuation of Deferred@1920111471 after retry RPC after error@1789842262) -> (continuation of Deferred@665091676 after retry RPC after error@1743440827) -> (continuation of Deferred@23446978 after retry RPC after error@500011739) -> (continuation of Deferred@429869472 after retry RPC after error@672792123) -> (continuation of Deferred@125662511 after retry RPC after error@1993831505)))
at org.junit.Assert.fail(Assert.java:89)
at org.junit.Assert.assertTrue(Assert.java:42)
at org.apache.kudu.client.TestKuduTransaction.testTxnKeepaliveRollingSwitchToOtherTxnManager(TestKuduTransaction.java:1214)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:569)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
at java.base/java.lang.Thread.run(Thread.java:840)